From 5458a42ad03aa4881ac198d0f8b2d1ae152b02c8 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 4 Apr 2019 19:42:09 -0500 Subject: [PATCH 01/16] add parser rule to catch old-style Range query --- pql/pql.peg | 1 + pql/pql.peg.go | 2622 +++++++++++++++++++++++--------------------- pql/pqlpeg_test.go | 4 + 3 files changed, 1396 insertions(+), 1231 deletions(-) diff --git a/pql/pql.peg b/pql/pql.peg index 8127ecc83..348396d5a 100644 --- a/pql/pql.peg +++ b/pql/pql.peg @@ -14,6 +14,7 @@ Call <- 'Set' {p.startCall("Set")} open col comma args (comma timestamp)? close / 'Store' {p.startCall("Store")} open Call comma arg close {p.endCall()} / 'TopN' {p.startCall("TopN")} open posfield (comma allargs)? close {p.endCall()} / 'Rows' {p.startCall("Rows")} open posfield (comma allargs)? close {p.endCall()} + / 'Range' {p.startCall("Range")} open field sp '=' sp value comma 'from='? {p.addField("from")} timestampfmt {p.addVal(buffer[begin:end])} comma 'to='? sp {p.addField("to")} timestampfmt {p.addVal(buffer[begin:end])} close {p.endCall()} / < IDENT > { p.startCall(buffer[begin:end] ) } open allargs comma? close { p.endCall() } allargs <- Call (comma Call)* (comma args)? / args / sp args <- arg (comma args)? sp diff --git a/pql/pql.peg.go b/pql/pql.peg.go index 8d960f95c..acec51d19 100644 --- a/pql/pql.peg.go +++ b/pql/pql.peg.go @@ -64,13 +64,13 @@ const ( ruleAction13 ruleAction14 ruleAction15 - rulePegText ruleAction16 ruleAction17 ruleAction18 ruleAction19 ruleAction20 ruleAction21 + rulePegText ruleAction22 ruleAction23 ruleAction24 @@ -101,6 +101,12 @@ const ( ruleAction49 ruleAction50 ruleAction51 + ruleAction52 + ruleAction53 + ruleAction54 + ruleAction55 + ruleAction56 + ruleAction57 ) var rul3s = [...]string{ @@ -153,13 +159,13 @@ var rul3s = [...]string{ "Action13", "Action14", "Action15", - "PegText", "Action16", "Action17", "Action18", "Action19", "Action20", "Action21", + "PegText", "Action22", "Action23", "Action24", @@ -190,6 +196,12 @@ var rul3s = [...]string{ "Action49", "Action50", "Action51", + "Action52", + "Action53", + "Action54", + "Action55", + "Action56", + "Action57", } type token32 struct { @@ -306,7 +318,7 @@ type PQL struct { Buffer string buffer []rune - rules [86]func() bool + rules [92]func() bool parse func(rule ...int) error reset func() Pretty bool @@ -431,77 +443,89 @@ func (p *PQL) Execute() { case ruleAction15: p.endCall() case ruleAction16: - p.startCall(buffer[begin:end]) + p.startCall("Range") case ruleAction17: - p.endCall() + p.addField("from") case ruleAction18: - p.addBTWN() + p.addVal(buffer[begin:end]) case ruleAction19: - p.addLTE() + p.addField("to") case ruleAction20: - p.addGTE() + p.addVal(buffer[begin:end]) case ruleAction21: - p.addEQ() + p.endCall() case ruleAction22: - p.addNEQ() - case ruleAction23: - p.addLT() - case ruleAction24: - p.addGT() - case ruleAction25: - p.startConditional() - case ruleAction26: - p.endConditional() - case ruleAction27: - p.condAdd(buffer[begin:end]) - case ruleAction28: - p.condAdd(buffer[begin:end]) - case ruleAction29: - p.condAdd(buffer[begin:end]) - case ruleAction30: - p.startList() - case ruleAction31: - p.endList() - case ruleAction32: - p.addVal(nil) - case ruleAction33: - p.addVal(true) - case ruleAction34: - p.addVal(false) - case ruleAction35: - p.addVal(buffer[begin:end]) - case ruleAction36: - p.addNumVal(buffer[begin:end]) - case ruleAction37: - p.addNumVal(buffer[begin:end]) - case ruleAction38: p.startCall(buffer[begin:end]) + case ruleAction23: + p.endCall() + case ruleAction24: + p.addBTWN() + case ruleAction25: + p.addLTE() + case ruleAction26: + p.addGTE() + case ruleAction27: + p.addEQ() + case ruleAction28: + p.addNEQ() + case ruleAction29: + p.addLT() + case ruleAction30: + p.addGT() + case ruleAction31: + p.startConditional() + case ruleAction32: + p.endConditional() + case ruleAction33: + p.condAdd(buffer[begin:end]) + case ruleAction34: + p.condAdd(buffer[begin:end]) + case ruleAction35: + p.condAdd(buffer[begin:end]) + case ruleAction36: + p.startList() + case ruleAction37: + p.endList() + case ruleAction38: + p.addVal(nil) case ruleAction39: - p.addVal(p.endCall()) + p.addVal(true) case ruleAction40: - p.addVal(buffer[begin:end]) + p.addVal(false) case ruleAction41: + p.addVal(buffer[begin:end]) + case ruleAction42: + p.addNumVal(buffer[begin:end]) + case ruleAction43: + p.addNumVal(buffer[begin:end]) + case ruleAction44: + p.startCall(buffer[begin:end]) + case ruleAction45: + p.addVal(p.endCall()) + case ruleAction46: + p.addVal(buffer[begin:end]) + case ruleAction47: s, _ := strconv.Unquote(buffer[begin:end]) p.addVal(s) - case ruleAction42: - p.addVal(buffer[begin:end]) - case ruleAction43: - p.addField(buffer[begin:end]) - case ruleAction44: - p.addPosStr("_field", buffer[begin:end]) - case ruleAction45: - p.addPosNum("_col", buffer[begin:end]) - case ruleAction46: - p.addPosStr("_col", buffer[begin:end]) - case ruleAction47: - p.addPosStr("_col", buffer[begin:end]) case ruleAction48: - p.addPosNum("_row", buffer[begin:end]) + p.addVal(buffer[begin:end]) case ruleAction49: - p.addPosStr("_row", buffer[begin:end]) + p.addField(buffer[begin:end]) case ruleAction50: - p.addPosStr("_row", buffer[begin:end]) + p.addPosStr("_field", buffer[begin:end]) case ruleAction51: + p.addPosNum("_col", buffer[begin:end]) + case ruleAction52: + p.addPosStr("_col", buffer[begin:end]) + case ruleAction53: + p.addPosStr("_col", buffer[begin:end]) + case ruleAction54: + p.addPosNum("_row", buffer[begin:end]) + case ruleAction55: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction56: + p.addPosStr("_row", buffer[begin:end]) + case ruleAction57: p.addPosStr("_timestamp", buffer[begin:end]) } @@ -614,7 +638,7 @@ func (p *PQL) Init() { position, tokenIndex = position0, tokenIndex0 return false }, - /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ('R' 'o' 'w' 's' Action14 open posfield (comma allargs)? close Action15) / ( Action16 open allargs comma? close Action17))> */ + /* 1 Call <- <(('S' 'e' 't' Action0 open col comma args (comma timestamp)? close Action1) / ('S' 'e' 't' 'R' 'o' 'w' 'A' 't' 't' 'r' 's' Action2 open posfield comma row comma args close Action3) / ('S' 'e' 't' 'C' 'o' 'l' 'u' 'm' 'n' 'A' 't' 't' 'r' 's' Action4 open col comma args close Action5) / ('C' 'l' 'e' 'a' 'r' Action6 open col comma args close Action7) / ('C' 'l' 'e' 'a' 'r' 'R' 'o' 'w' Action8 open arg close Action9) / ('S' 't' 'o' 'r' 'e' Action10 open Call comma arg close Action11) / ('T' 'o' 'p' 'N' Action12 open posfield (comma allargs)? close Action13) / ('R' 'o' 'w' 's' Action14 open posfield (comma allargs)? close Action15) / ('R' 'a' 'n' 'g' 'e' Action16 open field sp '=' sp value comma ('f' 'r' 'o' 'm' '=')? Action17 timestampfmt Action18 comma ('t' 'o' '=')? sp Action19 timestampfmt Action20 close Action21) / ( Action22 open allargs comma? close Action23))> */ func() bool { position5, tokenIndex5 := position, tokenIndex { @@ -663,7 +687,7 @@ func (p *PQL) Init() { add(rulePegText, position13) } { - add(ruleAction51, position) + add(ruleAction57, position) } add(ruletimestamp, position12) } @@ -749,7 +773,7 @@ func (p *PQL) Init() { add(rulePegText, position21) } { - add(ruleAction48, position) + add(ruleAction54, position) } goto l19 l20: @@ -770,7 +794,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction49, position) + add(ruleAction55, position) } goto l19 l23: @@ -791,7 +815,7 @@ func (p *PQL) Init() { } position++ { - add(ruleAction50, position) + add(ruleAction56, position) } } l19: @@ -1124,15 +1148,139 @@ func (p *PQL) Init() { goto l7 l46: position, tokenIndex = position7, tokenIndex7 + if buffer[position] != rune('R') { + goto l51 + } + position++ + if buffer[position] != rune('a') { + goto l51 + } + position++ + if buffer[position] != rune('n') { + goto l51 + } + position++ + if buffer[position] != rune('g') { + goto l51 + } + position++ + if buffer[position] != rune('e') { + goto l51 + } + position++ { - position51 := position + add(ruleAction16, position) + } + if !_rules[ruleopen]() { + goto l51 + } + if !_rules[rulefield]() { + goto l51 + } + if !_rules[rulesp]() { + goto l51 + } + if buffer[position] != rune('=') { + goto l51 + } + position++ + if !_rules[rulesp]() { + goto l51 + } + if !_rules[rulevalue]() { + goto l51 + } + if !_rules[rulecomma]() { + goto l51 + } + { + position53, tokenIndex53 := position, tokenIndex + if buffer[position] != rune('f') { + goto l53 + } + position++ + if buffer[position] != rune('r') { + goto l53 + } + position++ + if buffer[position] != rune('o') { + goto l53 + } + position++ + if buffer[position] != rune('m') { + goto l53 + } + position++ + if buffer[position] != rune('=') { + goto l53 + } + position++ + goto l54 + l53: + position, tokenIndex = position53, tokenIndex53 + } + l54: + { + add(ruleAction17, position) + } + if !_rules[ruletimestampfmt]() { + goto l51 + } + { + add(ruleAction18, position) + } + if !_rules[rulecomma]() { + goto l51 + } + { + position57, tokenIndex57 := position, tokenIndex + if buffer[position] != rune('t') { + goto l57 + } + position++ + if buffer[position] != rune('o') { + goto l57 + } + position++ + if buffer[position] != rune('=') { + goto l57 + } + position++ + goto l58 + l57: + position, tokenIndex = position57, tokenIndex57 + } + l58: + if !_rules[rulesp]() { + goto l51 + } + { + add(ruleAction19, position) + } + if !_rules[ruletimestampfmt]() { + goto l51 + } + { + add(ruleAction20, position) + } + if !_rules[ruleclose]() { + goto l51 + } + { + add(ruleAction21, position) + } + goto l7 + l51: + position, tokenIndex = position7, tokenIndex7 + { + position62 := position if !_rules[ruleIDENT]() { goto l5 } - add(rulePegText, position51) + add(rulePegText, position62) } { - add(ruleAction16, position) + add(ruleAction22, position) } if !_rules[ruleopen]() { goto l5 @@ -1141,20 +1289,20 @@ func (p *PQL) Init() { goto l5 } { - position53, tokenIndex53 := position, tokenIndex + position64, tokenIndex64 := position, tokenIndex if !_rules[rulecomma]() { - goto l53 + goto l64 } - goto l54 - l53: - position, tokenIndex = position53, tokenIndex53 + goto l65 + l64: + position, tokenIndex = position64, tokenIndex64 } - l54: + l65: if !_rules[ruleclose]() { goto l5 } { - add(ruleAction17, position) + add(ruleAction23, position) } } l7: @@ -1167,670 +1315,612 @@ func (p *PQL) Init() { }, /* 2 allargs <- <((Call (comma Call)* (comma args)?) / args / sp)> */ func() bool { - position56, tokenIndex56 := position, tokenIndex + position67, tokenIndex67 := position, tokenIndex { - position57 := position + position68 := position { - position58, tokenIndex58 := position, tokenIndex + position69, tokenIndex69 := position, tokenIndex if !_rules[ruleCall]() { - goto l59 + goto l70 } - l60: + l71: { - position61, tokenIndex61 := position, tokenIndex + position72, tokenIndex72 := position, tokenIndex if !_rules[rulecomma]() { - goto l61 + goto l72 } if !_rules[ruleCall]() { - goto l61 + goto l72 } - goto l60 - l61: - position, tokenIndex = position61, tokenIndex61 + goto l71 + l72: + position, tokenIndex = position72, tokenIndex72 } { - position62, tokenIndex62 := position, tokenIndex + position73, tokenIndex73 := position, tokenIndex if !_rules[rulecomma]() { - goto l62 + goto l73 } if !_rules[ruleargs]() { - goto l62 + goto l73 } - goto l63 - l62: - position, tokenIndex = position62, tokenIndex62 + goto l74 + l73: + position, tokenIndex = position73, tokenIndex73 } - l63: - goto l58 - l59: - position, tokenIndex = position58, tokenIndex58 + l74: + goto l69 + l70: + position, tokenIndex = position69, tokenIndex69 if !_rules[ruleargs]() { - goto l64 + goto l75 } - goto l58 - l64: - position, tokenIndex = position58, tokenIndex58 + goto l69 + l75: + position, tokenIndex = position69, tokenIndex69 if !_rules[rulesp]() { - goto l56 + goto l67 } } - l58: - add(ruleallargs, position57) + l69: + add(ruleallargs, position68) } return true - l56: - position, tokenIndex = position56, tokenIndex56 + l67: + position, tokenIndex = position67, tokenIndex67 return false }, /* 3 args <- <(arg (comma args)? sp)> */ func() bool { - position65, tokenIndex65 := position, tokenIndex + position76, tokenIndex76 := position, tokenIndex { - position66 := position + position77 := position if !_rules[rulearg]() { - goto l65 + goto l76 } { - position67, tokenIndex67 := position, tokenIndex + position78, tokenIndex78 := position, tokenIndex if !_rules[rulecomma]() { - goto l67 + goto l78 } if !_rules[ruleargs]() { - goto l67 + goto l78 } - goto l68 - l67: - position, tokenIndex = position67, tokenIndex67 + goto l79 + l78: + position, tokenIndex = position78, tokenIndex78 } - l68: + l79: if !_rules[rulesp]() { - goto l65 + goto l76 } - add(ruleargs, position66) + add(ruleargs, position77) } return true - l65: - position, tokenIndex = position65, tokenIndex65 + l76: + position, tokenIndex = position76, tokenIndex76 return false }, /* 4 arg <- <((field sp '=' sp value) / (field sp COND sp value) / conditional)> */ func() bool { - position69, tokenIndex69 := position, tokenIndex + position80, tokenIndex80 := position, tokenIndex { - position70 := position + position81 := position { - position71, tokenIndex71 := position, tokenIndex + position82, tokenIndex82 := position, tokenIndex if !_rules[rulefield]() { - goto l72 + goto l83 } if !_rules[rulesp]() { - goto l72 + goto l83 } if buffer[position] != rune('=') { - goto l72 + goto l83 } position++ if !_rules[rulesp]() { - goto l72 + goto l83 } if !_rules[rulevalue]() { - goto l72 + goto l83 } - goto l71 - l72: - position, tokenIndex = position71, tokenIndex71 + goto l82 + l83: + position, tokenIndex = position82, tokenIndex82 if !_rules[rulefield]() { - goto l73 + goto l84 } if !_rules[rulesp]() { - goto l73 + goto l84 } { - position74 := position + position85 := position { - position75, tokenIndex75 := position, tokenIndex + position86, tokenIndex86 := position, tokenIndex if buffer[position] != rune('>') { - goto l76 + goto l87 } position++ if buffer[position] != rune('<') { - goto l76 - } - position++ - { - add(ruleAction18, position) - } - goto l75 - l76: - position, tokenIndex = position75, tokenIndex75 - if buffer[position] != rune('<') { - goto l78 - } - position++ - if buffer[position] != rune('=') { - goto l78 - } - position++ - { - add(ruleAction19, position) - } - goto l75 - l78: - position, tokenIndex = position75, tokenIndex75 - if buffer[position] != rune('>') { - goto l80 - } - position++ - if buffer[position] != rune('=') { - goto l80 - } - position++ - { - add(ruleAction20, position) - } - goto l75 - l80: - position, tokenIndex = position75, tokenIndex75 - if buffer[position] != rune('=') { - goto l82 - } - position++ - if buffer[position] != rune('=') { - goto l82 - } - position++ - { - add(ruleAction21, position) - } - goto l75 - l82: - position, tokenIndex = position75, tokenIndex75 - if buffer[position] != rune('!') { - goto l84 - } - position++ - if buffer[position] != rune('=') { - goto l84 - } - position++ - { - add(ruleAction22, position) - } - goto l75 - l84: - position, tokenIndex = position75, tokenIndex75 - if buffer[position] != rune('<') { - goto l86 - } - position++ - { - add(ruleAction23, position) - } - goto l75 - l86: - position, tokenIndex = position75, tokenIndex75 - if buffer[position] != rune('>') { - goto l73 + goto l87 } position++ { add(ruleAction24, position) } - } - l75: - add(ruleCOND, position74) - } - if !_rules[rulesp]() { - goto l73 - } - if !_rules[rulevalue]() { - goto l73 - } - goto l71 - l73: - position, tokenIndex = position71, tokenIndex71 - { - position89 := position - { - add(ruleAction25, position) - } - if !_rules[rulecondint]() { - goto l69 - } - if !_rules[rulecondLT]() { - goto l69 - } - { - position91 := position + goto l86 + l87: + position, tokenIndex = position86, tokenIndex86 + if buffer[position] != rune('<') { + goto l89 + } + position++ + if buffer[position] != rune('=') { + goto l89 + } + position++ { - position92 := position - if !_rules[rulefieldExpr]() { - goto l69 - } - add(rulePegText, position92) + add(ruleAction25, position) } - if !_rules[rulesp]() { - goto l69 + goto l86 + l89: + position, tokenIndex = position86, tokenIndex86 + if buffer[position] != rune('>') { + goto l91 } + position++ + if buffer[position] != rune('=') { + goto l91 + } + position++ + { + add(ruleAction26, position) + } + goto l86 + l91: + position, tokenIndex = position86, tokenIndex86 + if buffer[position] != rune('=') { + goto l93 + } + position++ + if buffer[position] != rune('=') { + goto l93 + } + position++ + { + add(ruleAction27, position) + } + goto l86 + l93: + position, tokenIndex = position86, tokenIndex86 + if buffer[position] != rune('!') { + goto l95 + } + position++ + if buffer[position] != rune('=') { + goto l95 + } + position++ + { + add(ruleAction28, position) + } + goto l86 + l95: + position, tokenIndex = position86, tokenIndex86 + if buffer[position] != rune('<') { + goto l97 + } + position++ { add(ruleAction29, position) } - add(rulecondfield, position91) + goto l86 + l97: + position, tokenIndex = position86, tokenIndex86 + if buffer[position] != rune('>') { + goto l84 + } + position++ + { + add(ruleAction30, position) + } } - if !_rules[rulecondLT]() { - goto l69 + l86: + add(ruleCOND, position85) + } + if !_rules[rulesp]() { + goto l84 + } + if !_rules[rulevalue]() { + goto l84 + } + goto l82 + l84: + position, tokenIndex = position82, tokenIndex82 + { + position100 := position + { + add(ruleAction31, position) } if !_rules[rulecondint]() { - goto l69 + goto l80 + } + if !_rules[rulecondLT]() { + goto l80 } { - add(ruleAction26, position) + position102 := position + { + position103 := position + if !_rules[rulefieldExpr]() { + goto l80 + } + add(rulePegText, position103) + } + if !_rules[rulesp]() { + goto l80 + } + { + add(ruleAction35, position) + } + add(rulecondfield, position102) } - add(ruleconditional, position89) + if !_rules[rulecondLT]() { + goto l80 + } + if !_rules[rulecondint]() { + goto l80 + } + { + add(ruleAction32, position) + } + add(ruleconditional, position100) } } - l71: - add(rulearg, position70) + l82: + add(rulearg, position81) } return true - l69: - position, tokenIndex = position69, tokenIndex69 + l80: + position, tokenIndex = position80, tokenIndex80 return false }, - /* 5 COND <- <(('>' '<' Action18) / ('<' '=' Action19) / ('>' '=' Action20) / ('=' '=' Action21) / ('!' '=' Action22) / ('<' Action23) / ('>' Action24))> */ + /* 5 COND <- <(('>' '<' Action24) / ('<' '=' Action25) / ('>' '=' Action26) / ('=' '=' Action27) / ('!' '=' Action28) / ('<' Action29) / ('>' Action30))> */ nil, - /* 6 conditional <- <(Action25 condint condLT condfield condLT condint Action26)> */ + /* 6 conditional <- <(Action31 condint condLT condfield condLT condint Action32)> */ nil, - /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action27)> */ + /* 7 condint <- <(<(('-'? [1-9] [0-9]*) / '0')> sp Action33)> */ func() bool { - position97, tokenIndex97 := position, tokenIndex + position108, tokenIndex108 := position, tokenIndex { - position98 := position + position109 := position { - position99 := position + position110 := position { - position100, tokenIndex100 := position, tokenIndex + position111, tokenIndex111 := position, tokenIndex { - position102, tokenIndex102 := position, tokenIndex + position113, tokenIndex113 := position, tokenIndex if buffer[position] != rune('-') { - goto l102 + goto l113 } position++ - goto l103 - l102: - position, tokenIndex = position102, tokenIndex102 + goto l114 + l113: + position, tokenIndex = position113, tokenIndex113 } - l103: + l114: if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l101 + goto l112 } position++ - l104: + l115: { - position105, tokenIndex105 := position, tokenIndex + position116, tokenIndex116 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l105 + goto l116 } position++ - goto l104 - l105: - position, tokenIndex = position105, tokenIndex105 + goto l115 + l116: + position, tokenIndex = position116, tokenIndex116 } - goto l100 - l101: - position, tokenIndex = position100, tokenIndex100 + goto l111 + l112: + position, tokenIndex = position111, tokenIndex111 if buffer[position] != rune('0') { - goto l97 + goto l108 } position++ } - l100: - add(rulePegText, position99) + l111: + add(rulePegText, position110) } if !_rules[rulesp]() { - goto l97 + goto l108 } { - add(ruleAction27, position) + add(ruleAction33, position) } - add(rulecondint, position98) + add(rulecondint, position109) } return true - l97: - position, tokenIndex = position97, tokenIndex97 + l108: + position, tokenIndex = position108, tokenIndex108 return false }, - /* 8 condLT <- <(<(('<' '=') / '<')> sp Action28)> */ + /* 8 condLT <- <(<(('<' '=') / '<')> sp Action34)> */ func() bool { - position107, tokenIndex107 := position, tokenIndex + position118, tokenIndex118 := position, tokenIndex { - position108 := position + position119 := position { - position109 := position + position120 := position { - position110, tokenIndex110 := position, tokenIndex + position121, tokenIndex121 := position, tokenIndex if buffer[position] != rune('<') { - goto l111 + goto l122 } position++ if buffer[position] != rune('=') { - goto l111 + goto l122 } position++ - goto l110 - l111: - position, tokenIndex = position110, tokenIndex110 + goto l121 + l122: + position, tokenIndex = position121, tokenIndex121 if buffer[position] != rune('<') { - goto l107 + goto l118 } position++ } - l110: - add(rulePegText, position109) + l121: + add(rulePegText, position120) } if !_rules[rulesp]() { - goto l107 + goto l118 } { - add(ruleAction28, position) + add(ruleAction34, position) } - add(rulecondLT, position108) + add(rulecondLT, position119) } return true - l107: - position, tokenIndex = position107, tokenIndex107 + l118: + position, tokenIndex = position118, tokenIndex118 return false }, - /* 9 condfield <- <( sp Action29)> */ + /* 9 condfield <- <( sp Action35)> */ nil, - /* 10 value <- <(item / (lbrack Action30 list rbrack Action31))> */ + /* 10 value <- <(item / (lbrack Action36 list rbrack Action37))> */ func() bool { - position114, tokenIndex114 := position, tokenIndex + position125, tokenIndex125 := position, tokenIndex { - position115 := position + position126 := position { - position116, tokenIndex116 := position, tokenIndex + position127, tokenIndex127 := position, tokenIndex if !_rules[ruleitem]() { - goto l117 + goto l128 } - goto l116 - l117: - position, tokenIndex = position116, tokenIndex116 + goto l127 + l128: + position, tokenIndex = position127, tokenIndex127 { - position118 := position + position129 := position if buffer[position] != rune('[') { - goto l114 + goto l125 } position++ if !_rules[rulesp]() { - goto l114 + goto l125 } - add(rulelbrack, position118) - } - { - add(ruleAction30, position) - } - if !_rules[rulelist]() { - goto l114 - } - { - position120 := position - if !_rules[rulesp]() { - goto l114 - } - if buffer[position] != rune(']') { - goto l114 - } - position++ - if !_rules[rulesp]() { - goto l114 - } - add(rulerbrack, position120) - } - { - add(ruleAction31, position) - } - } - l116: - add(rulevalue, position115) - } - return true - l114: - position, tokenIndex = position114, tokenIndex114 - return false - }, - /* 11 list <- <(item (comma list)?)> */ - func() bool { - position122, tokenIndex122 := position, tokenIndex - { - position123 := position - if !_rules[ruleitem]() { - goto l122 - } - { - position124, tokenIndex124 := position, tokenIndex - if !_rules[rulecomma]() { - goto l124 - } - if !_rules[rulelist]() { - goto l124 - } - goto l125 - l124: - position, tokenIndex = position124, tokenIndex124 - } - l125: - add(rulelist, position123) - } - return true - l122: - position, tokenIndex = position122, tokenIndex122 - return false - }, - /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action32) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action33) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action34) / (timestampfmt Action35) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action36) / (<('-'? '.' [0-9]+)> Action37) / ( Action38 open allargs comma? close Action39) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action40) / (<('"' doublequotedstring '"')> Action41) / ('\'' '\'' Action42))> */ - func() bool { - position126, tokenIndex126 := position, tokenIndex - { - position127 := position - { - position128, tokenIndex128 := position, tokenIndex - if buffer[position] != rune('n') { - goto l129 - } - position++ - if buffer[position] != rune('u') { - goto l129 - } - position++ - if buffer[position] != rune('l') { - goto l129 - } - position++ - if buffer[position] != rune('l') { - goto l129 - } - position++ - { - position130, tokenIndex130 := position, tokenIndex - { - position131, tokenIndex131 := position, tokenIndex - if !_rules[rulecomma]() { - goto l132 - } - goto l131 - l132: - position, tokenIndex = position131, tokenIndex131 - if !_rules[rulesp]() { - goto l129 - } - if !_rules[ruleclose]() { - goto l129 - } - } - l131: - position, tokenIndex = position130, tokenIndex130 - } - { - add(ruleAction32, position) - } - goto l128 - l129: - position, tokenIndex = position128, tokenIndex128 - if buffer[position] != rune('t') { - goto l134 - } - position++ - if buffer[position] != rune('r') { - goto l134 - } - position++ - if buffer[position] != rune('u') { - goto l134 - } - position++ - if buffer[position] != rune('e') { - goto l134 - } - position++ - { - position135, tokenIndex135 := position, tokenIndex - { - position136, tokenIndex136 := position, tokenIndex - if !_rules[rulecomma]() { - goto l137 - } - goto l136 - l137: - position, tokenIndex = position136, tokenIndex136 - if !_rules[rulesp]() { - goto l134 - } - if !_rules[ruleclose]() { - goto l134 - } - } - l136: - position, tokenIndex = position135, tokenIndex135 - } - { - add(ruleAction33, position) - } - goto l128 - l134: - position, tokenIndex = position128, tokenIndex128 - if buffer[position] != rune('f') { - goto l139 - } - position++ - if buffer[position] != rune('a') { - goto l139 - } - position++ - if buffer[position] != rune('l') { - goto l139 - } - position++ - if buffer[position] != rune('s') { - goto l139 - } - position++ - if buffer[position] != rune('e') { - goto l139 - } - position++ - { - position140, tokenIndex140 := position, tokenIndex - { - position141, tokenIndex141 := position, tokenIndex - if !_rules[rulecomma]() { - goto l142 - } - goto l141 - l142: - position, tokenIndex = position141, tokenIndex141 - if !_rules[rulesp]() { - goto l139 - } - if !_rules[ruleclose]() { - goto l139 - } - } - l141: - position, tokenIndex = position140, tokenIndex140 - } - { - add(ruleAction34, position) - } - goto l128 - l139: - position, tokenIndex = position128, tokenIndex128 - if !_rules[ruletimestampfmt]() { - goto l144 - } - { - add(ruleAction35, position) - } - goto l128 - l144: - position, tokenIndex = position128, tokenIndex128 - { - position147 := position - { - position148, tokenIndex148 := position, tokenIndex - if buffer[position] != rune('-') { - goto l148 - } - position++ - goto l149 - l148: - position, tokenIndex = position148, tokenIndex148 - } - l149: - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l146 - } - position++ - l150: - { - position151, tokenIndex151 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l151 - } - position++ - goto l150 - l151: - position, tokenIndex = position151, tokenIndex151 - } - { - position152, tokenIndex152 := position, tokenIndex - if buffer[position] != rune('.') { - goto l152 - } - position++ - l154: - { - position155, tokenIndex155 := position, tokenIndex - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l155 - } - position++ - goto l154 - l155: - position, tokenIndex = position155, tokenIndex155 - } - goto l153 - l152: - position, tokenIndex = position152, tokenIndex152 - } - l153: - add(rulePegText, position147) + add(rulelbrack, position129) } { add(ruleAction36, position) } - goto l128 - l146: - position, tokenIndex = position128, tokenIndex128 + if !_rules[rulelist]() { + goto l125 + } + { + position131 := position + if !_rules[rulesp]() { + goto l125 + } + if buffer[position] != rune(']') { + goto l125 + } + position++ + if !_rules[rulesp]() { + goto l125 + } + add(rulerbrack, position131) + } + { + add(ruleAction37, position) + } + } + l127: + add(rulevalue, position126) + } + return true + l125: + position, tokenIndex = position125, tokenIndex125 + return false + }, + /* 11 list <- <(item (comma list)?)> */ + func() bool { + position133, tokenIndex133 := position, tokenIndex + { + position134 := position + if !_rules[ruleitem]() { + goto l133 + } + { + position135, tokenIndex135 := position, tokenIndex + if !_rules[rulecomma]() { + goto l135 + } + if !_rules[rulelist]() { + goto l135 + } + goto l136 + l135: + position, tokenIndex = position135, tokenIndex135 + } + l136: + add(rulelist, position134) + } + return true + l133: + position, tokenIndex = position133, tokenIndex133 + return false + }, + /* 12 item <- <(('n' 'u' 'l' 'l' &(comma / (sp close)) Action38) / ('t' 'r' 'u' 'e' &(comma / (sp close)) Action39) / ('f' 'a' 'l' 's' 'e' &(comma / (sp close)) Action40) / (timestampfmt Action41) / (<('-'? [0-9]+ ('.' [0-9]*)?)> Action42) / (<('-'? '.' [0-9]+)> Action43) / ( Action44 open allargs comma? close Action45) / (<([a-z] / [A-Z] / [0-9] / '-' / '_' / ':')+> Action46) / (<('"' doublequotedstring '"')> Action47) / ('\'' '\'' Action48))> */ + func() bool { + position137, tokenIndex137 := position, tokenIndex + { + position138 := position + { + position139, tokenIndex139 := position, tokenIndex + if buffer[position] != rune('n') { + goto l140 + } + position++ + if buffer[position] != rune('u') { + goto l140 + } + position++ + if buffer[position] != rune('l') { + goto l140 + } + position++ + if buffer[position] != rune('l') { + goto l140 + } + position++ + { + position141, tokenIndex141 := position, tokenIndex + { + position142, tokenIndex142 := position, tokenIndex + if !_rules[rulecomma]() { + goto l143 + } + goto l142 + l143: + position, tokenIndex = position142, tokenIndex142 + if !_rules[rulesp]() { + goto l140 + } + if !_rules[ruleclose]() { + goto l140 + } + } + l142: + position, tokenIndex = position141, tokenIndex141 + } + { + add(ruleAction38, position) + } + goto l139 + l140: + position, tokenIndex = position139, tokenIndex139 + if buffer[position] != rune('t') { + goto l145 + } + position++ + if buffer[position] != rune('r') { + goto l145 + } + position++ + if buffer[position] != rune('u') { + goto l145 + } + position++ + if buffer[position] != rune('e') { + goto l145 + } + position++ + { + position146, tokenIndex146 := position, tokenIndex + { + position147, tokenIndex147 := position, tokenIndex + if !_rules[rulecomma]() { + goto l148 + } + goto l147 + l148: + position, tokenIndex = position147, tokenIndex147 + if !_rules[rulesp]() { + goto l145 + } + if !_rules[ruleclose]() { + goto l145 + } + } + l147: + position, tokenIndex = position146, tokenIndex146 + } + { + add(ruleAction39, position) + } + goto l139 + l145: + position, tokenIndex = position139, tokenIndex139 + if buffer[position] != rune('f') { + goto l150 + } + position++ + if buffer[position] != rune('a') { + goto l150 + } + position++ + if buffer[position] != rune('l') { + goto l150 + } + position++ + if buffer[position] != rune('s') { + goto l150 + } + position++ + if buffer[position] != rune('e') { + goto l150 + } + position++ + { + position151, tokenIndex151 := position, tokenIndex + { + position152, tokenIndex152 := position, tokenIndex + if !_rules[rulecomma]() { + goto l153 + } + goto l152 + l153: + position, tokenIndex = position152, tokenIndex152 + if !_rules[rulesp]() { + goto l150 + } + if !_rules[ruleclose]() { + goto l150 + } + } + l152: + position, tokenIndex = position151, tokenIndex151 + } + { + add(ruleAction40, position) + } + goto l139 + l150: + position, tokenIndex = position139, tokenIndex139 + if !_rules[ruletimestampfmt]() { + goto l155 + } + { + add(ruleAction41, position) + } + goto l139 + l155: + position, tokenIndex = position139, tokenIndex139 { position158 := position { @@ -1844,10 +1934,6 @@ func (p *PQL) Init() { position, tokenIndex = position159, tokenIndex159 } l160: - if buffer[position] != rune('.') { - goto l157 - } - position++ if c := buffer[position]; c < rune('0') || c > rune('9') { goto l157 } @@ -1863,793 +1949,855 @@ func (p *PQL) Init() { l162: position, tokenIndex = position162, tokenIndex162 } + { + position163, tokenIndex163 := position, tokenIndex + if buffer[position] != rune('.') { + goto l163 + } + position++ + l165: + { + position166, tokenIndex166 := position, tokenIndex + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l166 + } + position++ + goto l165 + l166: + position, tokenIndex = position166, tokenIndex166 + } + goto l164 + l163: + position, tokenIndex = position163, tokenIndex163 + } + l164: add(rulePegText, position158) } { - add(ruleAction37, position) + add(ruleAction42, position) } - goto l128 + goto l139 l157: - position, tokenIndex = position128, tokenIndex128 + position, tokenIndex = position139, tokenIndex139 { - position165 := position - if !_rules[ruleIDENT]() { - goto l164 - } - add(rulePegText, position165) - } - { - add(ruleAction38, position) - } - if !_rules[ruleopen]() { - goto l164 - } - if !_rules[ruleallargs]() { - goto l164 - } - { - position167, tokenIndex167 := position, tokenIndex - if !_rules[rulecomma]() { - goto l167 - } - goto l168 - l167: - position, tokenIndex = position167, tokenIndex167 - } - l168: - if !_rules[ruleclose]() { - goto l164 - } - { - add(ruleAction39, position) - } - goto l128 - l164: - position, tokenIndex = position128, tokenIndex128 - { - position171 := position + position169 := position { - position174, tokenIndex174 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l175 - } - position++ - goto l174 - l175: - position, tokenIndex = position174, tokenIndex174 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l176 - } - position++ - goto l174 - l176: - position, tokenIndex = position174, tokenIndex174 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l177 - } - position++ - goto l174 - l177: - position, tokenIndex = position174, tokenIndex174 + position170, tokenIndex170 := position, tokenIndex if buffer[position] != rune('-') { - goto l178 - } - position++ - goto l174 - l178: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune('_') { - goto l179 - } - position++ - goto l174 - l179: - position, tokenIndex = position174, tokenIndex174 - if buffer[position] != rune(':') { goto l170 } position++ + goto l171 + l170: + position, tokenIndex = position170, tokenIndex170 } - l174: + l171: + if buffer[position] != rune('.') { + goto l168 + } + position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l168 + } + position++ l172: { position173, tokenIndex173 := position, tokenIndex - { - position180, tokenIndex180 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l181 - } - position++ - goto l180 - l181: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l182 - } - position++ - goto l180 - l182: - position, tokenIndex = position180, tokenIndex180 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l183 - } - position++ - goto l180 - l183: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('-') { - goto l184 - } - position++ - goto l180 - l184: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune('_') { - goto l185 - } - position++ - goto l180 - l185: - position, tokenIndex = position180, tokenIndex180 - if buffer[position] != rune(':') { - goto l173 - } - position++ + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l173 } - l180: + position++ goto l172 l173: position, tokenIndex = position173, tokenIndex173 } - add(rulePegText, position171) + add(rulePegText, position169) } { - add(ruleAction40, position) + add(ruleAction43, position) } - goto l128 - l170: - position, tokenIndex = position128, tokenIndex128 + goto l139 + l168: + position, tokenIndex = position139, tokenIndex139 { - position188 := position + position176 := position + if !_rules[ruleIDENT]() { + goto l175 + } + add(rulePegText, position176) + } + { + add(ruleAction44, position) + } + if !_rules[ruleopen]() { + goto l175 + } + if !_rules[ruleallargs]() { + goto l175 + } + { + position178, tokenIndex178 := position, tokenIndex + if !_rules[rulecomma]() { + goto l178 + } + goto l179 + l178: + position, tokenIndex = position178, tokenIndex178 + } + l179: + if !_rules[ruleclose]() { + goto l175 + } + { + add(ruleAction45, position) + } + goto l139 + l175: + position, tokenIndex = position139, tokenIndex139 + { + position182 := position + { + position185, tokenIndex185 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l186 + } + position++ + goto l185 + l186: + position, tokenIndex = position185, tokenIndex185 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l187 + } + position++ + goto l185 + l187: + position, tokenIndex = position185, tokenIndex185 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l188 + } + position++ + goto l185 + l188: + position, tokenIndex = position185, tokenIndex185 + if buffer[position] != rune('-') { + goto l189 + } + position++ + goto l185 + l189: + position, tokenIndex = position185, tokenIndex185 + if buffer[position] != rune('_') { + goto l190 + } + position++ + goto l185 + l190: + position, tokenIndex = position185, tokenIndex185 + if buffer[position] != rune(':') { + goto l181 + } + position++ + } + l185: + l183: + { + position184, tokenIndex184 := position, tokenIndex + { + position191, tokenIndex191 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l192 + } + position++ + goto l191 + l192: + position, tokenIndex = position191, tokenIndex191 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l193 + } + position++ + goto l191 + l193: + position, tokenIndex = position191, tokenIndex191 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l194 + } + position++ + goto l191 + l194: + position, tokenIndex = position191, tokenIndex191 + if buffer[position] != rune('-') { + goto l195 + } + position++ + goto l191 + l195: + position, tokenIndex = position191, tokenIndex191 + if buffer[position] != rune('_') { + goto l196 + } + position++ + goto l191 + l196: + position, tokenIndex = position191, tokenIndex191 + if buffer[position] != rune(':') { + goto l184 + } + position++ + } + l191: + goto l183 + l184: + position, tokenIndex = position184, tokenIndex184 + } + add(rulePegText, position182) + } + { + add(ruleAction46, position) + } + goto l139 + l181: + position, tokenIndex = position139, tokenIndex139 + { + position199 := position if buffer[position] != rune('"') { - goto l187 + goto l198 } position++ if !_rules[ruledoublequotedstring]() { - goto l187 + goto l198 } if buffer[position] != rune('"') { - goto l187 + goto l198 } position++ - add(rulePegText, position188) + add(rulePegText, position199) } { - add(ruleAction41, position) + add(ruleAction47, position) } - goto l128 - l187: - position, tokenIndex = position128, tokenIndex128 + goto l139 + l198: + position, tokenIndex = position139, tokenIndex139 if buffer[position] != rune('\'') { - goto l126 + goto l137 } position++ { - position190 := position + position201 := position if !_rules[rulesinglequotedstring]() { - goto l126 + goto l137 } - add(rulePegText, position190) + add(rulePegText, position201) } if buffer[position] != rune('\'') { - goto l126 + goto l137 } position++ { - add(ruleAction42, position) + add(ruleAction48, position) } } - l128: - add(ruleitem, position127) + l139: + add(ruleitem, position138) } return true - l126: - position, tokenIndex = position126, tokenIndex126 + l137: + position, tokenIndex = position137, tokenIndex137 return false }, /* 13 doublequotedstring <- <(('\\' '"') / ('\\' '\\') / (!'"' .))*> */ func() bool { { - position193 := position - l194: + position204 := position + l205: { - position195, tokenIndex195 := position, tokenIndex + position206, tokenIndex206 := position, tokenIndex { - position196, tokenIndex196 := position, tokenIndex + position207, tokenIndex207 := position, tokenIndex if buffer[position] != rune('\\') { - goto l197 + goto l208 } position++ if buffer[position] != rune('"') { - goto l197 + goto l208 } position++ - goto l196 - l197: - position, tokenIndex = position196, tokenIndex196 + goto l207 + l208: + position, tokenIndex = position207, tokenIndex207 if buffer[position] != rune('\\') { - goto l198 + goto l209 } position++ if buffer[position] != rune('\\') { - goto l198 + goto l209 } position++ - goto l196 - l198: - position, tokenIndex = position196, tokenIndex196 + goto l207 + l209: + position, tokenIndex = position207, tokenIndex207 { - position199, tokenIndex199 := position, tokenIndex + position210, tokenIndex210 := position, tokenIndex if buffer[position] != rune('"') { - goto l199 + goto l210 } position++ - goto l195 - l199: - position, tokenIndex = position199, tokenIndex199 + goto l206 + l210: + position, tokenIndex = position210, tokenIndex210 } if !matchDot() { - goto l195 + goto l206 } } - l196: - goto l194 - l195: - position, tokenIndex = position195, tokenIndex195 + l207: + goto l205 + l206: + position, tokenIndex = position206, tokenIndex206 } - add(ruledoublequotedstring, position193) + add(ruledoublequotedstring, position204) } return true }, /* 14 singlequotedstring <- <(('\\' '\'') / ('\\' '\\') / (!'\'' .))*> */ func() bool { { - position201 := position - l202: + position212 := position + l213: { - position203, tokenIndex203 := position, tokenIndex + position214, tokenIndex214 := position, tokenIndex { - position204, tokenIndex204 := position, tokenIndex + position215, tokenIndex215 := position, tokenIndex if buffer[position] != rune('\\') { - goto l205 + goto l216 } position++ if buffer[position] != rune('\'') { - goto l205 + goto l216 } position++ - goto l204 - l205: - position, tokenIndex = position204, tokenIndex204 + goto l215 + l216: + position, tokenIndex = position215, tokenIndex215 if buffer[position] != rune('\\') { - goto l206 + goto l217 } position++ if buffer[position] != rune('\\') { - goto l206 + goto l217 } position++ - goto l204 - l206: - position, tokenIndex = position204, tokenIndex204 + goto l215 + l217: + position, tokenIndex = position215, tokenIndex215 { - position207, tokenIndex207 := position, tokenIndex + position218, tokenIndex218 := position, tokenIndex if buffer[position] != rune('\'') { - goto l207 + goto l218 } position++ - goto l203 - l207: - position, tokenIndex = position207, tokenIndex207 + goto l214 + l218: + position, tokenIndex = position218, tokenIndex218 } if !matchDot() { - goto l203 + goto l214 } } - l204: - goto l202 - l203: - position, tokenIndex = position203, tokenIndex203 + l215: + goto l213 + l214: + position, tokenIndex = position214, tokenIndex214 } - add(rulesinglequotedstring, position201) + add(rulesinglequotedstring, position212) } return true }, /* 15 fieldExpr <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9] / '_' / '-')*)> */ - func() bool { - position208, tokenIndex208 := position, tokenIndex - { - position209 := position - { - position210, tokenIndex210 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l211 - } - position++ - goto l210 - l211: - position, tokenIndex = position210, tokenIndex210 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l208 - } - position++ - } - l210: - l212: - { - position213, tokenIndex213 := position, tokenIndex - { - position214, tokenIndex214 := position, tokenIndex - if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l215 - } - position++ - goto l214 - l215: - position, tokenIndex = position214, tokenIndex214 - if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l216 - } - position++ - goto l214 - l216: - position, tokenIndex = position214, tokenIndex214 - if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l217 - } - position++ - goto l214 - l217: - position, tokenIndex = position214, tokenIndex214 - if buffer[position] != rune('_') { - goto l218 - } - position++ - goto l214 - l218: - position, tokenIndex = position214, tokenIndex214 - if buffer[position] != rune('-') { - goto l213 - } - position++ - } - l214: - goto l212 - l213: - position, tokenIndex = position213, tokenIndex213 - } - add(rulefieldExpr, position209) - } - return true - l208: - position, tokenIndex = position208, tokenIndex208 - return false - }, - /* 16 field <- <(<(fieldExpr / reserved)> Action43)> */ func() bool { position219, tokenIndex219 := position, tokenIndex { position220 := position { - position221 := position - { - position222, tokenIndex222 := position, tokenIndex - if !_rules[rulefieldExpr]() { - goto l223 - } + position221, tokenIndex221 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { goto l222 - l223: - position, tokenIndex = position222, tokenIndex222 - { - position224 := position - { - position225, tokenIndex225 := position, tokenIndex - if buffer[position] != rune('_') { - goto l226 - } - position++ - if buffer[position] != rune('r') { - goto l226 - } - position++ - if buffer[position] != rune('o') { - goto l226 - } - position++ - if buffer[position] != rune('w') { - goto l226 - } - position++ - goto l225 - l226: - position, tokenIndex = position225, tokenIndex225 - if buffer[position] != rune('_') { - goto l227 - } - position++ - if buffer[position] != rune('c') { - goto l227 - } - position++ - if buffer[position] != rune('o') { - goto l227 - } - position++ - if buffer[position] != rune('l') { - goto l227 - } - position++ - goto l225 - l227: - position, tokenIndex = position225, tokenIndex225 - if buffer[position] != rune('_') { - goto l228 - } - position++ - if buffer[position] != rune('s') { - goto l228 - } - position++ - if buffer[position] != rune('t') { - goto l228 - } - position++ - if buffer[position] != rune('a') { - goto l228 - } - position++ - if buffer[position] != rune('r') { - goto l228 - } - position++ - if buffer[position] != rune('t') { - goto l228 - } - position++ - goto l225 - l228: - position, tokenIndex = position225, tokenIndex225 - if buffer[position] != rune('_') { - goto l229 - } - position++ - if buffer[position] != rune('e') { - goto l229 - } - position++ - if buffer[position] != rune('n') { - goto l229 - } - position++ - if buffer[position] != rune('d') { - goto l229 - } - position++ - goto l225 - l229: - position, tokenIndex = position225, tokenIndex225 - if buffer[position] != rune('_') { - goto l230 - } - position++ - if buffer[position] != rune('t') { - goto l230 - } - position++ - if buffer[position] != rune('i') { - goto l230 - } - position++ - if buffer[position] != rune('m') { - goto l230 - } - position++ - if buffer[position] != rune('e') { - goto l230 - } - position++ - if buffer[position] != rune('s') { - goto l230 - } - position++ - if buffer[position] != rune('t') { - goto l230 - } - position++ - if buffer[position] != rune('a') { - goto l230 - } - position++ - if buffer[position] != rune('m') { - goto l230 - } - position++ - if buffer[position] != rune('p') { - goto l230 - } - position++ - goto l225 - l230: - position, tokenIndex = position225, tokenIndex225 - if buffer[position] != rune('_') { - goto l219 - } - position++ - if buffer[position] != rune('f') { - goto l219 - } - position++ - if buffer[position] != rune('i') { - goto l219 - } - position++ - if buffer[position] != rune('e') { - goto l219 - } - position++ - if buffer[position] != rune('l') { - goto l219 - } - position++ - if buffer[position] != rune('d') { - goto l219 - } - position++ - } - l225: - add(rulereserved, position224) - } } + position++ + goto l221 l222: - add(rulePegText, position221) + position, tokenIndex = position221, tokenIndex221 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l219 + } + position++ } + l221: + l223: { - add(ruleAction43, position) + position224, tokenIndex224 := position, tokenIndex + { + position225, tokenIndex225 := position, tokenIndex + if c := buffer[position]; c < rune('a') || c > rune('z') { + goto l226 + } + position++ + goto l225 + l226: + position, tokenIndex = position225, tokenIndex225 + if c := buffer[position]; c < rune('A') || c > rune('Z') { + goto l227 + } + position++ + goto l225 + l227: + position, tokenIndex = position225, tokenIndex225 + if c := buffer[position]; c < rune('0') || c > rune('9') { + goto l228 + } + position++ + goto l225 + l228: + position, tokenIndex = position225, tokenIndex225 + if buffer[position] != rune('_') { + goto l229 + } + position++ + goto l225 + l229: + position, tokenIndex = position225, tokenIndex225 + if buffer[position] != rune('-') { + goto l224 + } + position++ + } + l225: + goto l223 + l224: + position, tokenIndex = position224, tokenIndex224 } - add(rulefield, position220) + add(rulefieldExpr, position220) } return true l219: position, tokenIndex = position219, tokenIndex219 return false }, - /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ - nil, - /* 18 posfield <- <( Action44)> */ + /* 16 field <- <(<(fieldExpr / reserved)> Action49)> */ func() bool { - position233, tokenIndex233 := position, tokenIndex + position230, tokenIndex230 := position, tokenIndex { - position234 := position + position231 := position { - position235 := position - if !_rules[rulefieldExpr]() { + position232 := position + { + position233, tokenIndex233 := position, tokenIndex + if !_rules[rulefieldExpr]() { + goto l234 + } goto l233 + l234: + position, tokenIndex = position233, tokenIndex233 + { + position235 := position + { + position236, tokenIndex236 := position, tokenIndex + if buffer[position] != rune('_') { + goto l237 + } + position++ + if buffer[position] != rune('r') { + goto l237 + } + position++ + if buffer[position] != rune('o') { + goto l237 + } + position++ + if buffer[position] != rune('w') { + goto l237 + } + position++ + goto l236 + l237: + position, tokenIndex = position236, tokenIndex236 + if buffer[position] != rune('_') { + goto l238 + } + position++ + if buffer[position] != rune('c') { + goto l238 + } + position++ + if buffer[position] != rune('o') { + goto l238 + } + position++ + if buffer[position] != rune('l') { + goto l238 + } + position++ + goto l236 + l238: + position, tokenIndex = position236, tokenIndex236 + if buffer[position] != rune('_') { + goto l239 + } + position++ + if buffer[position] != rune('s') { + goto l239 + } + position++ + if buffer[position] != rune('t') { + goto l239 + } + position++ + if buffer[position] != rune('a') { + goto l239 + } + position++ + if buffer[position] != rune('r') { + goto l239 + } + position++ + if buffer[position] != rune('t') { + goto l239 + } + position++ + goto l236 + l239: + position, tokenIndex = position236, tokenIndex236 + if buffer[position] != rune('_') { + goto l240 + } + position++ + if buffer[position] != rune('e') { + goto l240 + } + position++ + if buffer[position] != rune('n') { + goto l240 + } + position++ + if buffer[position] != rune('d') { + goto l240 + } + position++ + goto l236 + l240: + position, tokenIndex = position236, tokenIndex236 + if buffer[position] != rune('_') { + goto l241 + } + position++ + if buffer[position] != rune('t') { + goto l241 + } + position++ + if buffer[position] != rune('i') { + goto l241 + } + position++ + if buffer[position] != rune('m') { + goto l241 + } + position++ + if buffer[position] != rune('e') { + goto l241 + } + position++ + if buffer[position] != rune('s') { + goto l241 + } + position++ + if buffer[position] != rune('t') { + goto l241 + } + position++ + if buffer[position] != rune('a') { + goto l241 + } + position++ + if buffer[position] != rune('m') { + goto l241 + } + position++ + if buffer[position] != rune('p') { + goto l241 + } + position++ + goto l236 + l241: + position, tokenIndex = position236, tokenIndex236 + if buffer[position] != rune('_') { + goto l230 + } + position++ + if buffer[position] != rune('f') { + goto l230 + } + position++ + if buffer[position] != rune('i') { + goto l230 + } + position++ + if buffer[position] != rune('e') { + goto l230 + } + position++ + if buffer[position] != rune('l') { + goto l230 + } + position++ + if buffer[position] != rune('d') { + goto l230 + } + position++ + } + l236: + add(rulereserved, position235) + } } - add(rulePegText, position235) + l233: + add(rulePegText, position232) } { - add(ruleAction44, position) + add(ruleAction49, position) } - add(ruleposfield, position234) + add(rulefield, position231) } return true - l233: - position, tokenIndex = position233, tokenIndex233 + l230: + position, tokenIndex = position230, tokenIndex230 + return false + }, + /* 17 reserved <- <(('_' 'r' 'o' 'w') / ('_' 'c' 'o' 'l') / ('_' 's' 't' 'a' 'r' 't') / ('_' 'e' 'n' 'd') / ('_' 't' 'i' 'm' 'e' 's' 't' 'a' 'm' 'p') / ('_' 'f' 'i' 'e' 'l' 'd'))> */ + nil, + /* 18 posfield <- <( Action50)> */ + func() bool { + position244, tokenIndex244 := position, tokenIndex + { + position245 := position + { + position246 := position + if !_rules[rulefieldExpr]() { + goto l244 + } + add(rulePegText, position246) + } + { + add(ruleAction50, position) + } + add(ruleposfield, position245) + } + return true + l244: + position, tokenIndex = position244, tokenIndex244 return false }, /* 19 uint <- <(([1-9] [0-9]*) / '0')> */ func() bool { - position237, tokenIndex237 := position, tokenIndex + position248, tokenIndex248 := position, tokenIndex { - position238 := position + position249 := position { - position239, tokenIndex239 := position, tokenIndex + position250, tokenIndex250 := position, tokenIndex if c := buffer[position]; c < rune('1') || c > rune('9') { - goto l240 + goto l251 } position++ - l241: + l252: { - position242, tokenIndex242 := position, tokenIndex + position253, tokenIndex253 := position, tokenIndex if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l242 + goto l253 } position++ - goto l241 - l242: - position, tokenIndex = position242, tokenIndex242 + goto l252 + l253: + position, tokenIndex = position253, tokenIndex253 } - goto l239 - l240: - position, tokenIndex = position239, tokenIndex239 + goto l250 + l251: + position, tokenIndex = position250, tokenIndex250 if buffer[position] != rune('0') { - goto l237 + goto l248 } position++ } - l239: - add(ruleuint, position238) + l250: + add(ruleuint, position249) } return true - l237: - position, tokenIndex = position237, tokenIndex237 + l248: + position, tokenIndex = position248, tokenIndex248 return false }, - /* 20 col <- <(( Action45) / ('\'' '\'' Action46) / ('"' '"' Action47))> */ + /* 20 col <- <(( Action51) / ('\'' '\'' Action52) / ('"' '"' Action53))> */ func() bool { - position243, tokenIndex243 := position, tokenIndex + position254, tokenIndex254 := position, tokenIndex { - position244 := position + position255 := position { - position245, tokenIndex245 := position, tokenIndex + position256, tokenIndex256 := position, tokenIndex { - position247 := position + position258 := position if !_rules[ruleuint]() { - goto l246 + goto l257 } - add(rulePegText, position247) + add(rulePegText, position258) } { - add(ruleAction45, position) + add(ruleAction51, position) } - goto l245 - l246: - position, tokenIndex = position245, tokenIndex245 + goto l256 + l257: + position, tokenIndex = position256, tokenIndex256 if buffer[position] != rune('\'') { - goto l249 + goto l260 } position++ { - position250 := position + position261 := position if !_rules[rulesinglequotedstring]() { - goto l249 + goto l260 } - add(rulePegText, position250) + add(rulePegText, position261) } if buffer[position] != rune('\'') { - goto l249 + goto l260 } position++ { - add(ruleAction46, position) + add(ruleAction52, position) } - goto l245 - l249: - position, tokenIndex = position245, tokenIndex245 + goto l256 + l260: + position, tokenIndex = position256, tokenIndex256 if buffer[position] != rune('"') { - goto l243 + goto l254 } position++ { - position252 := position + position263 := position if !_rules[ruledoublequotedstring]() { - goto l243 + goto l254 } - add(rulePegText, position252) + add(rulePegText, position263) } if buffer[position] != rune('"') { - goto l243 + goto l254 } position++ { - add(ruleAction47, position) + add(ruleAction53, position) } } - l245: - add(rulecol, position244) + l256: + add(rulecol, position255) } return true - l243: - position, tokenIndex = position243, tokenIndex243 + l254: + position, tokenIndex = position254, tokenIndex254 return false }, - /* 21 row <- <(( Action48) / ('\'' '\'' Action49) / ('"' '"' Action50))> */ + /* 21 row <- <(( Action54) / ('\'' '\'' Action55) / ('"' '"' Action56))> */ nil, /* 22 open <- <('(' sp)> */ func() bool { - position255, tokenIndex255 := position, tokenIndex + position266, tokenIndex266 := position, tokenIndex { - position256 := position + position267 := position if buffer[position] != rune('(') { - goto l255 + goto l266 } position++ if !_rules[rulesp]() { - goto l255 + goto l266 } - add(ruleopen, position256) + add(ruleopen, position267) } return true - l255: - position, tokenIndex = position255, tokenIndex255 + l266: + position, tokenIndex = position266, tokenIndex266 return false }, /* 23 close <- <(')' sp)> */ func() bool { - position257, tokenIndex257 := position, tokenIndex + position268, tokenIndex268 := position, tokenIndex { - position258 := position + position269 := position if buffer[position] != rune(')') { - goto l257 + goto l268 } position++ if !_rules[rulesp]() { - goto l257 + goto l268 } - add(ruleclose, position258) + add(ruleclose, position269) } return true - l257: - position, tokenIndex = position257, tokenIndex257 + l268: + position, tokenIndex = position268, tokenIndex268 return false }, /* 24 sp <- <(' ' / '\t' / '\n')*> */ func() bool { { - position260 := position - l261: + position271 := position + l272: { - position262, tokenIndex262 := position, tokenIndex + position273, tokenIndex273 := position, tokenIndex { - position263, tokenIndex263 := position, tokenIndex + position274, tokenIndex274 := position, tokenIndex if buffer[position] != rune(' ') { - goto l264 + goto l275 } position++ - goto l263 - l264: - position, tokenIndex = position263, tokenIndex263 + goto l274 + l275: + position, tokenIndex = position274, tokenIndex274 if buffer[position] != rune('\t') { - goto l265 + goto l276 } position++ - goto l263 - l265: - position, tokenIndex = position263, tokenIndex263 + goto l274 + l276: + position, tokenIndex = position274, tokenIndex274 if buffer[position] != rune('\n') { - goto l262 + goto l273 } position++ } - l263: - goto l261 - l262: - position, tokenIndex = position262, tokenIndex262 + l274: + goto l272 + l273: + position, tokenIndex = position273, tokenIndex273 } - add(rulesp, position260) + add(rulesp, position271) } return true }, /* 25 comma <- <(sp ',' sp)> */ func() bool { - position266, tokenIndex266 := position, tokenIndex + position277, tokenIndex277 := position, tokenIndex { - position267 := position + position278 := position if !_rules[rulesp]() { - goto l266 + goto l277 } if buffer[position] != rune(',') { - goto l266 + goto l277 } position++ if !_rules[rulesp]() { - goto l266 + goto l277 } - add(rulecomma, position267) + add(rulecomma, position278) } return true - l266: - position, tokenIndex = position266, tokenIndex266 + l277: + position, tokenIndex = position277, tokenIndex277 return false }, /* 26 lbrack <- <('[' sp)> */ @@ -2658,207 +2806,207 @@ func (p *PQL) Init() { nil, /* 28 IDENT <- <(([a-z] / [A-Z]) ([a-z] / [A-Z] / [0-9])*)> */ func() bool { - position270, tokenIndex270 := position, tokenIndex + position281, tokenIndex281 := position, tokenIndex { - position271 := position + position282 := position { - position272, tokenIndex272 := position, tokenIndex + position283, tokenIndex283 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l273 + goto l284 } position++ - goto l272 - l273: - position, tokenIndex = position272, tokenIndex272 + goto l283 + l284: + position, tokenIndex = position283, tokenIndex283 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l270 + goto l281 } position++ } - l272: - l274: + l283: + l285: { - position275, tokenIndex275 := position, tokenIndex + position286, tokenIndex286 := position, tokenIndex { - position276, tokenIndex276 := position, tokenIndex + position287, tokenIndex287 := position, tokenIndex if c := buffer[position]; c < rune('a') || c > rune('z') { - goto l277 + goto l288 } position++ - goto l276 - l277: - position, tokenIndex = position276, tokenIndex276 + goto l287 + l288: + position, tokenIndex = position287, tokenIndex287 if c := buffer[position]; c < rune('A') || c > rune('Z') { - goto l278 + goto l289 } position++ - goto l276 - l278: - position, tokenIndex = position276, tokenIndex276 + goto l287 + l289: + position, tokenIndex = position287, tokenIndex287 if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l275 + goto l286 } position++ } - l276: - goto l274 - l275: - position, tokenIndex = position275, tokenIndex275 + l287: + goto l285 + l286: + position, tokenIndex = position286, tokenIndex286 } - add(ruleIDENT, position271) + add(ruleIDENT, position282) } return true - l270: - position, tokenIndex = position270, tokenIndex270 + l281: + position, tokenIndex = position281, tokenIndex281 return false }, /* 29 timestampbasicfmt <- <([0-9] [0-9] [0-9] [0-9] '-' ('0' / '1') [0-9] '-' [0-3] [0-9] 'T' [0-9] [0-9] ':' [0-9] [0-9])> */ func() bool { - position279, tokenIndex279 := position, tokenIndex + position290, tokenIndex290 := position, tokenIndex { - position280 := position + position291 := position if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ if buffer[position] != rune('-') { - goto l279 + goto l290 } position++ { - position281, tokenIndex281 := position, tokenIndex + position292, tokenIndex292 := position, tokenIndex if buffer[position] != rune('0') { - goto l282 + goto l293 } position++ - goto l281 - l282: - position, tokenIndex = position281, tokenIndex281 + goto l292 + l293: + position, tokenIndex = position292, tokenIndex292 if buffer[position] != rune('1') { - goto l279 + goto l290 } position++ } - l281: + l292: if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ if buffer[position] != rune('-') { - goto l279 + goto l290 } position++ if c := buffer[position]; c < rune('0') || c > rune('3') { - goto l279 + goto l290 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ if buffer[position] != rune('T') { - goto l279 + goto l290 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ if buffer[position] != rune(':') { - goto l279 + goto l290 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ if c := buffer[position]; c < rune('0') || c > rune('9') { - goto l279 + goto l290 } position++ - add(ruletimestampbasicfmt, position280) + add(ruletimestampbasicfmt, position291) } return true - l279: - position, tokenIndex = position279, tokenIndex279 + l290: + position, tokenIndex = position290, tokenIndex290 return false }, /* 30 timestampfmt <- <(('"' '"') / ('\'' '\'') / )> */ func() bool { - position283, tokenIndex283 := position, tokenIndex + position294, tokenIndex294 := position, tokenIndex { - position284 := position + position295 := position { - position285, tokenIndex285 := position, tokenIndex + position296, tokenIndex296 := position, tokenIndex if buffer[position] != rune('"') { - goto l286 + goto l297 } position++ { - position287 := position + position298 := position if !_rules[ruletimestampbasicfmt]() { - goto l286 + goto l297 } - add(rulePegText, position287) + add(rulePegText, position298) } if buffer[position] != rune('"') { - goto l286 + goto l297 } position++ - goto l285 - l286: - position, tokenIndex = position285, tokenIndex285 + goto l296 + l297: + position, tokenIndex = position296, tokenIndex296 if buffer[position] != rune('\'') { - goto l288 + goto l299 } position++ { - position289 := position + position300 := position if !_rules[ruletimestampbasicfmt]() { - goto l288 + goto l299 } - add(rulePegText, position289) + add(rulePegText, position300) } if buffer[position] != rune('\'') { - goto l288 + goto l299 } position++ - goto l285 - l288: - position, tokenIndex = position285, tokenIndex285 + goto l296 + l299: + position, tokenIndex = position296, tokenIndex296 { - position290 := position + position301 := position if !_rules[ruletimestampbasicfmt]() { - goto l283 + goto l294 } - add(rulePegText, position290) + add(rulePegText, position301) } } - l285: - add(ruletimestampfmt, position284) + l296: + add(ruletimestampfmt, position295) } return true - l283: - position, tokenIndex = position283, tokenIndex283 + l294: + position, tokenIndex = position294, tokenIndex294 return false }, - /* 31 timestamp <- <( Action51)> */ + /* 31 timestamp <- <( Action57)> */ nil, /* 33 Action0 <- <{p.startCall("Set")}> */ nil, @@ -2892,78 +3040,90 @@ func (p *PQL) Init() { nil, /* 48 Action15 <- <{p.endCall()}> */ nil, + /* 49 Action16 <- <{p.startCall("Range")}> */ nil, - /* 50 Action16 <- <{ p.startCall(buffer[begin:end] ) }> */ + /* 50 Action17 <- <{p.addField("from")}> */ nil, - /* 51 Action17 <- <{ p.endCall() }> */ + /* 51 Action18 <- <{p.addVal(buffer[begin:end])}> */ nil, - /* 52 Action18 <- <{ p.addBTWN() }> */ + /* 52 Action19 <- <{p.addField("to")}> */ nil, - /* 53 Action19 <- <{ p.addLTE() }> */ + /* 53 Action20 <- <{p.addVal(buffer[begin:end])}> */ nil, - /* 54 Action20 <- <{ p.addGTE() }> */ + /* 54 Action21 <- <{p.endCall()}> */ nil, - /* 55 Action21 <- <{ p.addEQ() }> */ nil, - /* 56 Action22 <- <{ p.addNEQ() }> */ + /* 56 Action22 <- <{ p.startCall(buffer[begin:end] ) }> */ nil, - /* 57 Action23 <- <{ p.addLT() }> */ + /* 57 Action23 <- <{ p.endCall() }> */ nil, - /* 58 Action24 <- <{ p.addGT() }> */ + /* 58 Action24 <- <{ p.addBTWN() }> */ nil, - /* 59 Action25 <- <{p.startConditional()}> */ + /* 59 Action25 <- <{ p.addLTE() }> */ nil, - /* 60 Action26 <- <{p.endConditional()}> */ + /* 60 Action26 <- <{ p.addGTE() }> */ nil, - /* 61 Action27 <- <{p.condAdd(buffer[begin:end])}> */ + /* 61 Action27 <- <{ p.addEQ() }> */ nil, - /* 62 Action28 <- <{p.condAdd(buffer[begin:end])}> */ + /* 62 Action28 <- <{ p.addNEQ() }> */ nil, - /* 63 Action29 <- <{p.condAdd(buffer[begin:end])}> */ + /* 63 Action29 <- <{ p.addLT() }> */ nil, - /* 64 Action30 <- <{ p.startList() }> */ + /* 64 Action30 <- <{ p.addGT() }> */ nil, - /* 65 Action31 <- <{ p.endList() }> */ + /* 65 Action31 <- <{p.startConditional()}> */ nil, - /* 66 Action32 <- <{ p.addVal(nil) }> */ + /* 66 Action32 <- <{p.endConditional()}> */ nil, - /* 67 Action33 <- <{ p.addVal(true) }> */ + /* 67 Action33 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 68 Action34 <- <{ p.addVal(false) }> */ + /* 68 Action34 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 69 Action35 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 69 Action35 <- <{p.condAdd(buffer[begin:end])}> */ nil, - /* 70 Action36 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 70 Action36 <- <{ p.startList() }> */ nil, - /* 71 Action37 <- <{ p.addNumVal(buffer[begin:end]) }> */ + /* 71 Action37 <- <{ p.endList() }> */ nil, - /* 72 Action38 <- <{ p.startCall(buffer[begin:end]) }> */ + /* 72 Action38 <- <{ p.addVal(nil) }> */ nil, - /* 73 Action39 <- <{ p.addVal(p.endCall()) }> */ + /* 73 Action39 <- <{ p.addVal(true) }> */ nil, - /* 74 Action40 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 74 Action40 <- <{ p.addVal(false) }> */ nil, - /* 75 Action41 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ + /* 75 Action41 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 76 Action42 <- <{ p.addVal(buffer[begin:end]) }> */ + /* 76 Action42 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 77 Action43 <- <{ p.addField(buffer[begin:end]) }> */ + /* 77 Action43 <- <{ p.addNumVal(buffer[begin:end]) }> */ nil, - /* 78 Action44 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ + /* 78 Action44 <- <{ p.startCall(buffer[begin:end]) }> */ nil, - /* 79 Action45 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + /* 79 Action45 <- <{ p.addVal(p.endCall()) }> */ nil, - /* 80 Action46 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 80 Action46 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 81 Action47 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + /* 81 Action47 <- <{ s, _ := strconv.Unquote(buffer[begin:end]); p.addVal(s) }> */ nil, - /* 82 Action48 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + /* 82 Action48 <- <{ p.addVal(buffer[begin:end]) }> */ nil, - /* 83 Action49 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 83 Action49 <- <{ p.addField(buffer[begin:end]) }> */ nil, - /* 84 Action50 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + /* 84 Action50 <- <{ p.addPosStr("_field", buffer[begin:end]) }> */ nil, - /* 85 Action51 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ + /* 85 Action51 <- <{p.addPosNum("_col", buffer[begin:end])}> */ + nil, + /* 86 Action52 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + nil, + /* 87 Action53 <- <{p.addPosStr("_col", buffer[begin:end])}> */ + nil, + /* 88 Action54 <- <{p.addPosNum("_row", buffer[begin:end])}> */ + nil, + /* 89 Action55 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 90 Action56 <- <{p.addPosStr("_row", buffer[begin:end])}> */ + nil, + /* 91 Action57 <- <{p.addPosStr("_timestamp", buffer[begin:end])}> */ nil, } p.rules = _rules diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 54f73b809..8492d3531 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -267,6 +267,10 @@ func TestPEGWorking(t *testing.T) { my-frame =9)`, ncalls: 1}, + { + name: "OldRange", + input: "Range(blah=1, 2019-04-07T00:00, 2019-08-07T00:00)", + ncalls: 1}, } for i, test := range tests { From 55d9d49f2fe9496098918c4dede57edcda2da382 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 5 Apr 2019 07:52:25 -0500 Subject: [PATCH 02/16] add executor test for deprecated range query style --- executor_test.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/executor_test.go b/executor_test.go index a36ddcf72..06133738c 100644 --- a/executor_test.go +++ b/executor_test.go @@ -1720,6 +1720,17 @@ func TestExecutor_Execute_Range_Deprecated(t *testing.T) { t.Fatalf("unexpected columns: %+v", columns) } }) + + rq2 := []string{ + `Range(f=1, 1999-12-31T00:00, 2002-01-01T03:00)`, + } + responses = runCallTest(t, writeQuery, rq2, + nil, pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YMDH"))) + t.Run("OldRange", func(t *testing.T) { + if columns := responses[0].Results[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) + } + }) }) t.Run("RowIDColumnKey", func(t *testing.T) { From bd085e0a21b938708cf0c5b5477ba4cd399f9f71 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Mon, 1 Apr 2019 20:23:34 -0500 Subject: [PATCH 03/16] simply setting list of values with *N methods --- roaring/roaring.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/roaring/roaring.go b/roaring/roaring.go index fa2d09bcb..a8ccd81e2 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -3667,15 +3667,9 @@ func (op *op) apply(b *Bitmap) (changed bool) { case opTypeRemove: return b.remove(op.value) case opTypeAddBatch: - for _, v := range op.values { - nc := b.DirectAdd(v) - changed = nc || changed - } + changed = b.DirectAddN(op.values...) > 0 case opTypeRemoveBatch: - for _, v := range op.values { - nc := b.remove(v) - changed = nc || changed - } + changed = b.DirectRemoveN(op.values...) > 0 default: panic(fmt.Sprintf("invalid op type: %d", op.typ)) } From d5cfe880f7e064f9d760c7797871191c2663824d Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 5 Apr 2019 14:06:20 -0500 Subject: [PATCH 04/16] address race condition by getting cluster nodes with lock needed an unlocked version of sendsync for use within the cluster, so also implemented that. Added a number of tests trying to reproduce the issue, but was not able to. Not sure it's worth keeping the new tests. --- cluster.go | 24 +++++- server.go | 3 +- server/cluster_test.go | 183 +++++++++++++++++++++++++++++++++++++++++ server/server_test.go | 47 +++++++++++ utils_internal_test.go | 16 +++- 5 files changed, 268 insertions(+), 5 deletions(-) diff --git a/cluster.go b/cluster.go index 9ee35f4fa..fc3f024fd 100644 --- a/cluster.go +++ b/cluster.go @@ -307,7 +307,7 @@ func (c *cluster) setCoordinator(n *Node) error { _ = c.unprotectedUpdateCoordinator(n) c.mu.Unlock() // Send the update coordinator message to all nodes. - err := c.broadcaster.SendSync( + err := c.unprotectedSendSync( &UpdateCoordinatorMessage{ New: n, }) @@ -316,7 +316,25 @@ func (c *cluster) setCoordinator(n *Node) error { } // Broadcast cluster status. - return c.broadcaster.SendSync(c.status()) + return c.unprotectedSendSync(c.status()) +} + +// unprotectedSendSync is used in place of c.broadcaster.SendSync (which is +// Server.SendSync) because Server.SendSync needs to obtain a cluster lock to +// get the list of nodes. TODO: the reference loop from +// Server->cluster->broadcaster(Server) will likely continue to cause confusion +// and should be refactored. +func (c *cluster) unprotectedSendSync(m Message) error { + var eg errgroup.Group + for _, node := range c.nodes { + node := node + // Don't send to myself. + if node.ID == c.Node.ID { + continue + } + eg.Go(func() error { return c.broadcaster.SendTo(node, m) }) + } + return eg.Wait() } // updateCoordinator updates this nodes Coordinator value as well as @@ -1083,7 +1101,7 @@ func (c *cluster) unprotectedSetStateAndBroadcast(state string) error { } // Broadcast cluster status changes to the cluster. status := c.unprotectedStatus() - return c.broadcaster.SendSync(status) // TODO fix c.Status + return c.unprotectedSendSync(status) // TODO fix c.Status } diff --git a/server.go b/server.go index 5a311918f..3e989e530 100644 --- a/server.go +++ b/server.go @@ -589,7 +589,8 @@ func (s *Server) SendSync(m Message) error { return fmt.Errorf("marshaling message: %v", err) } msg = append([]byte{getMessageType(m)}, msg...) - for _, node := range s.cluster.nodes { + + for _, node := range s.cluster.Nodes() { node := node // Don't forward the message to ourselves. if s.uri == node.URI { diff --git a/server/cluster_test.go b/server/cluster_test.go index 617bf6cad..a0cd83f1b 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -309,6 +309,189 @@ func TestClusterResize_AddNode(t *testing.T) { }) } +// Ensure that adding a node correctly resizes the cluster. +func TestClusterResize_AddNodeConcurrentIndex(t *testing.T) { + t.Run("WithIndex", func(t *testing.T) { + // Configure node0 + m0 := test.MustRunCluster(t, 1)[0] + defer m0.Close() + + seed := m0.GossipAddress() + + // Create a client for each node. + client0 := m0.Client() + + // Create indexes and fields on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateField(context.Background(), "i", "f"); err != nil { + t.Fatal(err) + } + + errc := make(chan error) + go func() { + _, err := m0.API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{}) + errc <- err + }() + + // Configure node1 + m1 := test.NewCommandNode(false) + m1.Config.Gossip.Port = "0" + m1.Config.Gossip.Seeds = []string{seed} + err := m1.Start() + if err != nil { + t.Fatalf("starting second main: %v", err) + } + defer m1.Close() + + if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) + } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) + } + + if err := <-errc; err != nil { + t.Fatalf("error from index creation: %v", err) + } + }) + t.Run("ContinuousShards", func(t *testing.T) { + // Configure node0 + m0 := test.MustRunCluster(t, 1)[0] + defer m0.Close() + + seed := m0.GossipAddress() + + // Create a client for each node. + client0 := m0.Client() + + // Create indexes and fields on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateField(context.Background(), "i", "f"); err != nil { + t.Fatal(err) + } + + // Write data on first node. + if _, err := m0.Query("i", "", ` + Set(1, f=1) + Set(1300000, f=1) + `); err != nil { + t.Fatal(err) + } + + // exp is the expected result for the Row queries that follow. + exp := `{"results":[{"attrs":{},"columns":[1,1300000]}]}` + "\n" + + // Verify the data exists on the single node. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + + // Configure node1 + m1 := test.NewCommandNode(false) + m1.Config.Gossip.Port = "0" + m1.Config.Gossip.Seeds = []string{seed} + err := m1.Start() + if err != nil { + t.Fatalf("starting second main: %v", err) + } + errc := make(chan error, 1) + go func() { + _, err := m0.API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{}) + errc <- err + }() + defer m1.Close() + + if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) + } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) + } + + // Verify the data exists on both nodes. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + if res, err := m1.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + }) + t.Run("SkippedShard", func(t *testing.T) { + // Configure node0 + m0 := test.MustRunCluster(t, 1)[0] + defer m0.Close() + + seed := m0.GossipAddress() + + // Create a client for each node. + client0 := m0.Client() + + // Create indexes and fields on one node. + if err := client0.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { + t.Fatal(err) + } else if err := client0.CreateField(context.Background(), "i", "f"); err != nil { + t.Fatal(err) + } + + // Write data on first node. Note that no data is placed on shard 1. + if _, err := m0.Query("i", "", ` + Set(1, f=1) + Set(2400000, f=1) + `); err != nil { + t.Fatal(err) + } + + // exp is the expected result for the Row queries that follow. + exp := `{"results":[{"attrs":{},"columns":[1,2400000]}]}` + "\n" + + // Verify the data exists on the single node. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + + // Configure node1 + m1 := test.NewCommandNode(false) + m1.Config.Gossip.Port = "0" + m1.Config.Gossip.Seeds = []string{seed} + errc := make(chan error, 1) + go func() { + _, err := m0.API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{}) + errc <- err + }() + err := m1.Start() + if err != nil { + t.Fatalf("starting second main: %v", err) + } + defer m1.Close() + + if !checkClusterState(m0, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node0 cluster state: %s", m0.API.State()) + } else if !checkClusterState(m1, pilosa.ClusterStateNormal, 1000) { + t.Fatalf("unexpected node1 cluster state: %s", m1.API.State()) + } + + // Verify the data exists on both nodes. + if res, err := m0.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + if res, err := m1.Query("i", "", `Row(f=1)`); err != nil { + t.Fatal(err) + } else if res != exp { + t.Fatalf("unexpected result: %s", res) + } + }) +} + // Ensure that redundant gossip seeds are used func TestCluster_GossipMembership(t *testing.T) { t.Run("Node0Down", func(t *testing.T) { diff --git a/server/server_test.go b/server/server_test.go index 3b6be7efa..1cc3bd87f 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -630,6 +630,53 @@ func TestRemoveNodeAfterItDies(t *testing.T) { } } +func TestRemoveConcurrentIndexCreation(t *testing.T) { + cluster := test.MustNewCluster(t, 3) + for _, c := range cluster { + c.Config.Cluster.ReplicaN = 2 + } + err := cluster.Start() + if err != nil { + t.Fatalf("starting cluster: %v", err) + } + + var wait = true + for wait { + wait = false + for _, node := range cluster { + if node.API.State() != pilosa.ClusterStateNormal { + wait = true + } + } + time.Sleep(time.Millisecond * 1) + } + + errc := make(chan error) + go func() { + _, err := cluster[0].API.CreateIndex(context.Background(), "blah", pilosa.IndexOptions{}) + errc <- err + }() + + if _, err := cluster[0].API.RemoveNode(cluster[2].API.Node().ID); err != nil { + t.Fatalf("removing node: %v", err) + } + + for i := 0; cluster[0].API.State() != pilosa.ClusterStateNormal; i++ { + time.Sleep(time.Millisecond) + if i > 10 { + t.Fatalf("expected state to be DEGRADED, but got %s", cluster[0].API.State()) + } + } + + hosts := cluster[0].API.Hosts(context.Background()) + if len(hosts) != 2 { + t.Fatalf("unexpected hosts: %v", hosts) + } + if err := <-errc; err != nil { + t.Fatalf("error from index creation: %v", err) + } +} + // Ensure program imports timestamps as UTC. func TestMain_ImportTimestamp(t *testing.T) { m := test.MustRunCommand() diff --git a/utils_internal_test.go b/utils_internal_test.go index 016d5276f..bf98f08a8 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -338,7 +338,7 @@ func (bcast) SendAsync(Message) error { return nil } -// SendTo is a test implemenetation of Broadcaster SendTo method. +// SendTo is a test implementation of Broadcaster SendTo method. func (b bcast) SendTo(to *Node, m Message) error { switch obj := m.(type) { case *ResizeInstruction: @@ -349,6 +349,20 @@ func (b bcast) SendTo(to *Node, m Message) error { case *ResizeInstructionComplete: coord := b.t.clusterByID(to.ID) go coord.markResizeInstructionComplete(obj) + case *ClusterStatus: + // Apply the send message to the node. + for _, c := range b.t.Clusters { + if c.Node.ID == to.ID { + c.mergeClusterStatus(obj) + } + } + b.t.mu.RLock() + if obj.State == ClusterStateNormal && b.t.resizing { + close(b.t.resizeDone) + } + b.t.mu.RUnlock() + default: + panic(fmt.Sprintf("message not handled:\n%#v\n", obj)) } return nil } From 79968e5d2ffdbdfbb89235d591c37af57c59d446 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 5 Apr 2019 14:50:26 -0500 Subject: [PATCH 05/16] fix unlocked access to cluster.nodes --- cluster.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cluster.go b/cluster.go index fc3f024fd..4012381fd 100644 --- a/cluster.go +++ b/cluster.go @@ -296,16 +296,16 @@ func (c *cluster) unprotectedIsCoordinator() bool { // nodes with its version of Cluster.Status. func (c *cluster) setCoordinator(n *Node) error { c.mu.Lock() + defer c.mu.Unlock() // Verify that the new Coordinator value matches // this node. if c.Node.ID != n.ID { - c.mu.Unlock() return fmt.Errorf("coordinator node does not match this node") } // Update IsCoordinator on all nodes (locally). _ = c.unprotectedUpdateCoordinator(n) - c.mu.Unlock() + // Send the update coordinator message to all nodes. err := c.unprotectedSendSync( &UpdateCoordinatorMessage{ @@ -316,7 +316,7 @@ func (c *cluster) setCoordinator(n *Node) error { } // Broadcast cluster status. - return c.unprotectedSendSync(c.status()) + return c.unprotectedSendSync(c.unprotectedStatus()) } // unprotectedSendSync is used in place of c.broadcaster.SendSync (which is From 7b436a4e30085315cdb04d50e59a93ba93d2ecce Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Sat, 6 Apr 2019 09:46:39 -0500 Subject: [PATCH 06/16] remove now-unused cluster.status method --- cluster.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cluster.go b/cluster.go index 4012381fd..204a1016b 100644 --- a/cluster.go +++ b/cluster.go @@ -551,12 +551,6 @@ func (c *cluster) determineClusterState() (clusterState string) { return ClusterStateStarting } -func (c *cluster) status() *ClusterStatus { - c.mu.RLock() - defer c.mu.RUnlock() - return c.unprotectedStatus() -} - // unprotectedStatus returns the the cluster's status including what nodes it contains, its ID, and current state. func (c *cluster) unprotectedStatus() *ClusterStatus { return &ClusterStatus{ From 09e4e2091523e74750300b20e368eeb297fb6433 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Sun, 7 Apr 2019 16:42:22 -0500 Subject: [PATCH 07/16] replace memberlist dep with patched fork fixes (hopefully) race condidtion which plagues our CI builds --- go.mod | 2 ++ go.sum | 6 ++++++ 2 files changed, 8 insertions(+) diff --git a/go.mod b/go.mod index a360af2fe..2cfa54cd8 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/pilosa/pilosa +replace github.com/hashicorp/memberlist => github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 + require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 diff --git a/go.sum b/go.sum index 0500034a8..e77168275 100644 --- a/go.sum +++ b/go.sum @@ -61,6 +61,12 @@ github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/pilosa/memberlist v0.1.3 h1:6am86S+mnY3zKPmH5yHtTqdNpqH/KjxF6WSHk95Msyo= +github.com/pilosa/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07 h1:f1Xp66+XJjfFAqnhX3T/4X3ItZK1H+r9neBnK+nV1ec= +github.com/pilosa/memberlist v0.1.4-0.20190406170317-7e5a340efc07/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= +github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108 h1:6QjQrHgdgVR7nnbzPwJwZ1dliUdjYtFi6ma50GtLOwA= +github.com/pilosa/memberlist v0.1.4-0.20190408132233-ff8741fd3108/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= From b95f739b07d2ddeca055b2919590e21dd7ed5527 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 10 Apr 2019 15:07:07 +0300 Subject: [PATCH 08/16] Updated import and client libraries docs --- docs/administration.md | 9 +- docs/client-libraries.md | 282 +-------------------------------------- 2 files changed, 12 insertions(+), 279 deletions(-) diff --git a/docs/administration.md b/docs/administration.md index 4bfa5470a..021bd5535 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -56,6 +56,11 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` +Official Pilosa client libraries support importing data. You can find the corresponding documentation at: +* [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md) +* [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md) +* [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md) + ##### Importing Integer Values If you are using [integer](../data-model/#bsi-range-encoding) field values, the CSV file should be in the format `Column,Value`. @@ -283,13 +288,13 @@ Each Pilosa cluster is configured by default to share anonymous usage details wi - **NumViews:** Number of views in the Cluster. - **OpenFiles:** Open file handle count. - **GoRoutines:** Go routine count. - + You can opt-out of the Pilosa diagnostics reporting by setting the command line configuration option `--metric.diagnostics=false`, the `PILOSA_METRIC_DIAGNOSTICS` environment variable, or the TOML configuration file `[metric]` `diagnostics` option. ### Metrics Pilosa can be configured to emit metrics pertaining to its internal processes in one of two formats: Expvar or StatsD. Metric recording is disabled by default. -The metrics configuration options are: +The metrics configuration options are: - [Host](../configuration/#metric-host): specify host that receives metric events - [Poll Interval](../configuration/#metric-poll-interval): specify polling interval for runtime metrics diff --git a/docs/client-libraries.md b/docs/client-libraries.md index ac008def4..9492f7693 100644 --- a/docs/client-libraries.md +++ b/docs/client-libraries.md @@ -10,281 +10,9 @@ nav = [ ## Client Libraries -This section contains example code for client libraries in several languages. Please remember that when modeling your data in Pilosa, it is best to keep row and column ids sequential. It is best to avoid using the output of a hash or randomly distributed ids with Pilosa. +We have the following official client libraries. You can find more information in their repositories: +* [Go client repository](https://github.com/pilosa/go-pilosa) +* [Java client repository](https://github.com/pilosa/java-pilosa) +* [Python client repository](https://github.com/pilosa/python-pilosa) -### Go - -You can find the Go client library for Pilosa at our [Go Pilosa Repository](https://github.com/pilosa/go-pilosa). Check out its [README](https://github.com/pilosa/go-pilosa/blob/master/README.md) for more information and installation instructions. - -We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. - -Error handling has been omitted in the example below for brevity. - -```go -package main - -import ( - "fmt" - - "github.com/pilosa/go-pilosa" -) - -func main() { - // We will just use the default client which assumes the server is at http://localhost:10101 - client := pilosa.DefaultClient() - - // Let's load the schema from the server. - // Note that, for this example the schema should be created beforehand - // and the stargazer data should be imported. - // See the Getting Started repository: https://github.com/pilosa/getting-started/ - schema, err := client.Schema() - if err != nil { - // Most calls will return an error value. - // You should handle them appropriately. - // We will just terminate the program in this case. - // Error handling was left out for brevity in the rest of the code. - panic(err) - } - - // We need to refer to indexes and fields before we can use them in a query. - repository := schema.Index("repository") - stargazer := repository.Field("stargazer") - language := repository.Field("language") - - var response *pilosa.QueryResponse - - // Which repositories did user 14 star: - response, _ = client.Query(stargazer.Row(14)) - fmt.Println("User 14 starred: ", response.Result().Row().Columns) - - // What are the top 5 languages in the sample data? - response, err = client.Query(language.TopN(5)) - languageIDs := []uint64{} - for _, item := range response.Result().CountItems() { - languageIDs = append(languageIDs, item.ID) - } - fmt.Println("Top 5 languages: ", languageIDs) - - // Which repositories were starred by both user 14 and 19: - response, _ = client.Query( - repository.Intersect( - stargazer.Row(14), - stargazer.Row(19))) - fmt.Println("Both user 14 and 19 starred:", response.Result().Row().Columns) - - // Which repositories were starred by user 14 or 19: - response, _ = client.Query( - repository.Union( - stargazer.Row(14), - stargazer.Row(19))) - fmt.Println("User 14 or 19 starred:", response.Result().Row().Columns) - - // Which repositories were starred by user 14 or 19 and were written in language 1: - response, _ = client.Query( - repository.Intersect( - repository.Union( - stargazer.Row(14), - stargazer.Row(19), - ), - language.Row(1))) - fmt.Println("User 14 or 19 starred, written in language 1:", response.Result().Row().Columns) - - // Set user 99999 as a stargazer for repository 77777? - client.Query(stargazer.Set(99999, 77777)) -} -``` - -Running the above program should produce output like this: -``` -User 14 starred: [1 2 3 362 368 391 396 409 416 430 436 450 454 460 461 464 466 469 470 483 484 486 490 491 503 504 514] -Top 5 languages: [5 1 4 9 13] -Both user 14 and 19 starred: [2 3 362 396 416 461 464 466 470 486] -User 14 or 19 starred: [1 2 3 361 362 368 376 377 378 382 386 388 391 396 398 400 409 411 412 416 426 428 430 435 436 450 452 453 454 456 460 461 464 465 466 469 470 483 484 486 487 489 490 491 500 503 504 505 512 514] -User 14 or 19 starred, written in language 1: [1 2 362 368 382 386 416 426 435 456 461 483 500 503 504 514] -``` - -### Python - -You can find the Python client library for Pilosa at our [Python Pilosa Repository](https://github.com/pilosa/python-pilosa). Check out its [README](https://github.com/pilosa/python-pilosa/blob/master/README.md) or [readthedocs](https://pilosa.readthedocs.io/en/latest/) for more information and installation instructions. - -We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. - -Error handling has been omitted in the example below for brevity. - -```python -from __future__ import print_function -from pilosa import Index, Client, PilosaError, TimeQuantum - -# We will just use the default client which assumes the server is at http://localhost:10101 -client = Client() - -# Let's load the schema from the server. -# Note that, for this example the schema should be created beforehand -# and the stargazer data should be imported. -# See the Getting Started repository: https://github.com/pilosa/getting-started/ - -# Let's create Index and Field objects, which will contain the settings -# for the corresponding indexes and fields. -try: - schema = client.schema() -except PilosaError as e: - # Most calls will raise an exception on errors. - # You should handle them appropriately. - # We will just terminate the program in this case. - raise SystemExit(e) - -# We need to refer to indexes and fields before we can use them in a query. -repository = schema.index("repository") -stargazer = repository.field("stargazer") -language = repository.field("language") - -# Which repositories did user 8 star: -repository_ids = client.query(stargazer.row(14)).result.row.columns -print("User 8 starred: ", repository_ids) - -# What are the top 5 languages in the sample data: -top_languages = client.query(language.topn(5)).result.count_items -print("Top 5 languages: ", [item.id for item in top_languages]) - -# Which repositories were starred by both user 14 and 19: -query = repository.intersect( - stargazer.row(14), - stargazer.row(19) -) -mutually_starred = client.query(query).result.row.columns -print("Both user 14 and 19 starred:", mutually_starred) - -# Which repositories were starred by user 14 or 19: -query = repository.union( - stargazer.row(14), - stargazer.row(19) -) -either_starred = client.query(query).result.row.columns -print("User 14 or 19 starred:", either_starred) - -# Which repositories were starred by user 14 or 19 and were written in language 1: -query = repository.intersect( - repository.union( - stargazer.row(14), - stargazer.row(19) - ), - language.row(1) -) -mutually_starred = client.query(query).result.row.columns -print("User 14 or 19 starred, written in language 1:", mutually_starred) - -# Set user 99999 as a stargazer for repository 77777 -client.query(stargazer.set(99999, 77777)) -``` - -Running the above program should produce output like this: -``` -('User 8 starred: ', [1L, 2L, 3L, 362L, 368L, 391L, 396L, 409L, 416L, 430L, 436L, 450L, 454L, 460L, 461L, 464L, 466L, 469L, 470L, 483L, 484L, 486L, 490L, 491L, 503L, 504L, 514L]) -('Top 5 languages: ', [5L, 1L, 4L, 9L, 13L]) -('Both user 14 and 19 starred:', [2L, 3L, 362L, 396L, 416L, 461L, 464L, 466L, 470L, 486L]) -('User 14 or 19 starred:', [1L, 2L, 3L, 361L, 362L, 368L, 376L, 377L, 378L, 382L, 386L, 388L, 391L, 396L, 398L, 400L, 409L, 411L, 412L, 416L, 426L, 428L, 430L, 435L, 436L, 450L, 452L, 453L, 454L, 456L, 460L, 461L, 464L, 465L, 466L, 469L, 470L, 483L, 484L, 486L, 487L, 489L, 490L, 491L, 500L, 503L, 504L, 505L, 512L, 514L]) -('User 14 or 19 starred, written in language 1:', [1L, 2L, 362L, 368L, 382L, 386L, 416L, 426L, 435L, 456L, 461L, 483L, 500L, 503L, 504L, 514L]) -``` - -### Java - -You can find the Java client library for Pilosa at our [Java Pilosa Repository](https://github.com/pilosa/java-pilosa). Check out its [README](https://github.com/pilosa/java-pilosa/blob/master/README.md) for more information and installation instructions. - -We are going to use the index you have created in the [Getting Started](../getting-started/) section. Before carrying on, make sure that example index is created, sample stargazer data is imported and Pilosa server is running on the default address: `http://localhost:10101`. - -Error handling has been omitted in the example below for brevity. - -```java -import com.pilosa.client.*; -import com.pilosa.client.orm.*; -import com.pilosa.client.exceptions.PilosaException; - -import java.util.ArrayList; -import java.util.List; - -public class StarTrace { - public static void main(String[] args) { - // We will just use the default client which assumes the server is at http://localhost:10101 - PilosaClient client = PilosaClient.defaultClient(); - - // Let's load the schema from the server. - Schema schema; - try { - schema = client.readSchema(); - } - catch (PilosaException ex) { - // Most calls will return an error value. - // You should handle them appropriately. - // We will just terminate the program in this case. - throw new RuntimeException(ex); - } - - // We need to refer to indexes and fields before we can use them in a query. - Index repository = schema.index("repository"); - Field stargazer = repository.field("stargazer"); - Field language = repository.field("language"); - - QueryResponse response; - QueryResult result; - PqlQuery query; - List repositoryIDs; - - // Which repositories did user 14 star: - response = client.query(stargazer.row(14)); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("User 14 starred: " + repositoryIDs); - - // What are the top 5 languages in the sample data: - response = client.query(language.topN(5)); - List top_languages = response.getResult().getCountItems(); - List languageIDs = new ArrayList(); - for (CountResultItem item : top_languages) { - languageIDs.add(item.getID()); - } - - System.out.println("Top Languages: " +languageIDs); - - // Which repositories were starred by both user 14 and 19: - query = repository.intersect( - stargazer.row(14), - stargazer.row(19) - ); - response = client.query(query); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("Both user 14 and 19 starred: " + repositoryIDs); - - // Which repositories were starred by user 14 or 19: - query = repository.union( - stargazer.row(14), - stargazer.row(19) - ); - response = client.query(query); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("User 14 or 19 starred: " + repositoryIDs); - - // Which repositories were starred by user 14 or 19 and were written in language 1: - query = repository.intersect( - repository.union( - stargazer.row(14), - stargazer.row(19) - ), - language.row(1) - ); - response = client.query(query); - repositoryIDs = response.getResult().getRow().getColumns(); - System.out.println("User 14 or 19 starred, written in language 1: " + repositoryIDs); - - // Set user 99999 as a stargazer for repository 77777: - client.query(stargazer.set(99999, 77777)); - } -} -``` - -Running the above program should produce output like this: -``` -User 14 starred: [1, 2, 3, 362, 368, 391, 396, 409, 416, 430, 436, 450, 454, 460, 461, 464, 466, 469, 470, 483, 484, 486, 490, 491, 503, 504, 514] -Top Languages: [5, 1, 4, 9, 13] -Both user 14 and 19 starred: [2, 3, 362, 396, 416, 461, 464, 466, 470, 486] -User 14 or 19 starred: [1, 2, 3, 361, 362, 368, 376, 377, 378, 382, 386, 388, 391, 396, 398, 400, 409, 411, 412, 416, 426, 428, 430, 435, 436, 450, 452, 453, 454, 456, 460, 461, 464, 465, 466, 469, 470, 483, 484, 486, 487, 489, 490, 491, 500, 503, 504, 505, 512, 514] -User 14 or 19 starred, written in language 1: [1, 2, 362, 368, 382, 386, 416, 426, 435, 456, 461, 483, 500, 503, 504, 514] -``` +Check out our [Getting Started](https://github.com/pilosa/getting-started) repository for sample code for the official clients. From cc4de714732364267862e98c0fbec58c1ed2416b Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 10 Apr 2019 17:14:47 +0300 Subject: [PATCH 09/16] updated examples section --- docs/examples.md | 159 +++++++---------------------------------------- 1 file changed, 24 insertions(+), 135 deletions(-) diff --git a/docs/examples.md b/docs/examples.md index 692cd7e39..c7476a722 100644 --- a/docs/examples.md +++ b/docs/examples.md @@ -83,10 +83,10 @@ lfm := pdk.LinearFloatMapper{ `Min` and `Max` define the linear function, and `Res` determines the maximum allowed value for the output row ID - we chose these values to produce a “round to nearest integer” behavior. Other predefined mappers have their own specific parameters, usually two or three. -This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the BitMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Frame`). TODO update so this makes sense +This mapper function is the core operation, but we need a few other pieces to define the overall process, which is encapsulated in the ColumnMapper object. This object defines which field(s) of the input data source to use (`Fields`), how to parse them (`Parsers`), what mapping to use (`Mapper`), and the name of the field to use (`Field`). ```go -pdk.BitMapper{ - Frame: "dist_miles", +pdk.ColumnMapper{ + Field: "dist_miles", Mapper: lfm, Parsers: []pdk.Parser{pdk.FloatParser{}}, Fields: []int{fields["trip_distance"]}, @@ -107,9 +107,9 @@ These same objects are represented in the JSON definition file: "Res": 3600 } ], - "BitMappers": [ + "ColumnMappers": [ { - "Frame": "dist_miles", + "Field": "dist_miles", "Mapper": { "Name": "lfm0" }, @@ -122,9 +122,9 @@ These same objects are represented in the JSON definition file: } ``` -Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of BitMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names (in the source data) to column indices (in Pilosa). We use these names in the BitMapper definitions to keep things human-readable. +Here, we define a list of Mappers, each including a name, which we use to refer to the mapper later, in the list of ColumnMappers. We can also do this with Parsers, but a few simple Parsers that need no configuration are available by default. We also have a list of Fields, which is simply a map of field names (in the source data) to column indices (in Pilosa). We use these names in the ColumnMapper definitions to keep things human-readable. -**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The BitMapper definition is very similar to the previous one. +**total_amount_dollars:** Here we use the rounding mapping again, so each row represents rides with a total cost that rounds to the row's ID. The ColumnMapper definition is very similar to the previous one. **passenger_count:** This column contains small integers, so we use one of the simplest possible mappings: the column value is the row ID. @@ -132,7 +132,7 @@ Here, we define a list of Mappers, each including a name, which we use to refer When working with a composite data type like a timestamp, there are plenty of mapping options. In this case, we expect to see interesting periodic trends, so we want to encode the cyclic components of time in a way that allows us to look at them independently during analysis. -We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day". +We do this by storing time data in four separate fields for each timestamp: one each for the year, month, day, and time of day. The first three are mapped directly. For example, a ride with a date of 2015/06/24 will have a bit set in row 2015 of field "year", row 6 of field "month", and row 24 of field "day". We might continue this pattern with hours, minutes, and seconds, but we don't have much use for that level of precision here, so instead we use a "bucketing" approach. That is, we pick a resolution (30 minutes), divide the day into buckets of that size, and create a row for each one. So a ride with a time of 6:45AM has a bit set in row 13 of field "time_of_day". @@ -189,16 +189,26 @@ TopN(pickup_grid_id) Average of `total_amount` per `passenger_count` can be computed with some postprocessing. We use a small number of `TopN` calls to retrieve counts of rides by passenger_count, then use those counts to compute an average. ```python -queries = '' +import pilosa + +client = pilosa.Client() +schema = client.schema() +taxi = schema.index("taxi") +passenger_count = taxi.field("passenger_count") +total_amount_dollars = taxi.field("total_amount_dollars") + +queries = [] pcounts = range(10) for i in pcounts: - queries += "TopN(Row(passenger_count=%d), total_amount_dollars)" % i + queries.append(total_amount_dollars.topn(passenger_count.row(i)) +query = taxi.batch_query(**queries) +results = client.query(query) resp = requests.post(qurl, data=queries) average_amounts = [] -for pcount, topn in zip(pcounts, resp.json()['results']): - wsum = sum([r['count'] * r['key'] for r in topn]) - count = sum([r['count'] for r in topn]) +for pcount, result in zip(pcounts, resp.results): + wsum = sum([r.count * r.id for r in result.count_items]) + count = sum([r.count for r in result.count_items]) average_amounts.append(float(wsum)/count) ``` @@ -206,127 +216,6 @@ for pcount, topn in zip(pcounts, resp.json()['results']): Note that the BSI-powered Sum query now provides an alternative approach to this kind of query. + From 73cc49770cba74b6c81a35798176e7f2cfafcac4 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 10 Apr 2019 17:17:55 +0300 Subject: [PATCH 10/16] suggest client libraries for import --- docs/administration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index 021bd5535..42c47e517 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -56,7 +56,7 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` -Official Pilosa client libraries support importing data. You can find the corresponding documentation at: +We suggest importing the data using official Pilosa client libraries. You can find the corresponding documentation at: * [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md) * [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md) * [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md) From 2fd62872223512650c856655cc6407cf6f2dcbf5 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 10 Apr 2019 17:20:02 +0300 Subject: [PATCH 11/16] recommend --- docs/administration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index 42c47e517..23929c267 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -56,7 +56,7 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` -We suggest importing the data using official Pilosa client libraries. You can find the corresponding documentation at: +We recommend importing the data using official Pilosa client libraries. You can find the corresponding documentation at: * [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md) * [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md) * [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md) From 1e7208d60ff5b1c4f8a23044f99419d66969d35e Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Wed, 10 Apr 2019 17:25:21 +0300 Subject: [PATCH 12/16] the --- docs/administration.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/administration.md b/docs/administration.md index 23929c267..33fa73104 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -56,7 +56,7 @@ When importing large datasets remember it is much faster to pre sort the data by pilosa import --sort -i project -f stargazer project-stargazer.csv ``` -We recommend importing the data using official Pilosa client libraries. You can find the corresponding documentation at: +We recommend importing data using official Pilosa client libraries. You can find the corresponding documentation at: * [Go client imports documentation](https://github.com/pilosa/go-pilosa/blob/master/docs/imports-exports.md) * [Java client imports documentation](https://github.com/pilosa/java-pilosa/blob/master/docs/imports.md) * [Python client imports documentation](https://github.com/pilosa/python-pilosa/blob/master/docs/imports.md) From 992a075cfb89512da9bcc4ee7dd706a237d0cf50 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 10 Apr 2019 15:27:12 -0500 Subject: [PATCH 13/16] Return empty result set when query empty. Fixes #1840 --- handler.go | 21 +++++++++++---------- server/handler_test.go | 8 ++++++++ 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/handler.go b/handler.go index 988435a07..5a452ce96 100644 --- a/handler.go +++ b/handler.go @@ -45,18 +45,19 @@ type QueryResponse struct { // MarshalJSON marshals QueryResponse into a JSON-encoded byte slice func (resp *QueryResponse) MarshalJSON() ([]byte, error) { - var output struct { - Results []interface{} `json:"results,omitempty"` - ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"` - Err string `json:"error,omitempty"` - } - output.Results = resp.Results - output.ColumnAttrSets = resp.ColumnAttrSets - if resp.Err != nil { - output.Err = resp.Err.Error() + return json.Marshal(struct { + Err string `json:"error"` + }{Err: resp.Err.Error()}) } - return json.Marshal(output) + + return json.Marshal(struct { + Results []interface{} `json:"results"` + ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"` + }{ + Results: resp.Results, + ColumnAttrSets: resp.ColumnAttrSets, + }) } type Handler interface { diff --git a/server/handler_test.go b/server/handler_test.go index 701d019ae..ed7a2bed6 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -445,6 +445,14 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("Query empty", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(""))) + if body := w.Body.String(); body != `{"results":[]}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + }) + t.Run("Method not allowed", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/query", nil)) From 7bb6fdffcb6d5f7323b3dbb30ce8582b27dcab51 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 10 Apr 2019 14:28:18 -0500 Subject: [PATCH 14/16] Add correct content type to query responses. Fixes #1873 --- http/handler.go | 4 +++- server/handler_test.go | 4 ++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/http/handler.go b/http/handler.go index 8cc059035..3226eee50 100644 --- a/http/handler.go +++ b/http/handler.go @@ -964,10 +964,12 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*pilosa.QueryRequest, er } // writeQueryResponse writes the response from the executor to w. -func (h *Handler) writeQueryResponse(w io.Writer, r *http.Request, resp *pilosa.QueryResponse) error { +func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *pilosa.QueryResponse) error { if !validHeaderAcceptJSON(r.Header) { + w.Header().Set("Content-Type", "application/protobuf") return h.writeProtobufQueryResponse(w, resp) } + w.Header().Set("Content-Type", "application/json") return h.writeJSONQueryResponse(w, resp) } diff --git a/server/handler_test.go b/server/handler_test.go index ed7a2bed6..f45aa2422 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -248,6 +248,8 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatalf("unexpected status code: %d", w.Code) } else if body := w.Body.String(); body != `{"results":[2]}`+"\n" { t.Fatalf("unexpected body: %q", body) + } else if w.Header().Get("Content-Type") != "application/json" { + t.Fatalf("unexpected header: %q", w.Header().Get("Content-Type")) } }) @@ -286,6 +288,8 @@ func TestHandler_Endpoints(t *testing.T) { t.Fatal(err) } else if rt, ok := resp.Results[0].(uint64); !ok || rt != 3 { t.Fatalf("unexpected response type: %#v", resp.Results[0]) + } else if w.Header().Get("Content-Type") != "application/protobuf" { + t.Fatalf("unexpected header: %q", w.Header().Get("Content-Type")) } }) From 5f079163cbf2c5570f580f5d7eaa18ff69c7fde2 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 10 Apr 2019 16:30:40 -0500 Subject: [PATCH 15/16] validate (and panic) on duplicate PQL arguments --- pql/ast.go | 18 +++++++++++++++- pql/pqlpeg_test.go | 52 ++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/pql/ast.go b/pql/ast.go index d78ad2828..36d6e35fb 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -103,7 +103,9 @@ func (q *Query) endConditional() { func (q *Query) addField(field string) { elem := q.lastCallStackElem() - if elem == nil || elem.lastField != "" { + if elem == nil { + panic(fmt.Sprintf("addField called with '%s' while element is nil", field)) + } else if elem.lastField != "" { panic(fmt.Sprintf("addField called with '%s' while field is not empty, it's: %s", field, elem.lastField)) } elem.lastField = field @@ -112,6 +114,15 @@ func (q *Query) addField(field string) { } } +// validateArgField ensures that field does not already +// exist as a key in the Args map before adding the new +// key/value. +func (q *Query) validateArgField(elem *callStackElem) { + if _, exists := elem.call.Args[elem.lastField]; exists { + panic(fmt.Sprintf("multiple instances of argument '%s' provided", elem.lastField)) + } +} + func (q *Query) addVal(val interface{}) { elem := q.lastCallStackElem() if elem == nil || elem.lastField == "" { @@ -123,11 +134,13 @@ func (q *Query) addVal(val interface{}) { return } if elem.lastCond != ILLEGAL { + q.validateArgField(elem) // case 1 elem.call.Args[elem.lastField] = &Condition{ Op: elem.lastCond, Value: val, } } else { + q.validateArgField(elem) // case 2 elem.call.Args[elem.lastField] = val } elem.lastField = "" @@ -162,11 +175,13 @@ func (q *Query) addNumVal(val string) { } return } else if elem.lastCond != ILLEGAL { + q.validateArgField(elem) // case 3 elem.call.Args[elem.lastField] = &Condition{ Op: elem.lastCond, Value: ival, } } else { + q.validateArgField(elem) // case 4 elem.call.Args[elem.lastField] = ival } elem.lastField = "" @@ -175,6 +190,7 @@ func (q *Query) addNumVal(val string) { func (q *Query) startList() { elem := q.lastCallStackElem() + q.validateArgField(elem) // case 5 if elem.lastCond != ILLEGAL { elem.call.Args[elem.lastField] = &Condition{ Op: elem.lastCond, diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index 8492d3531..f8a6597b5 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -676,3 +676,55 @@ func TestPQLDeepEquality(t *testing.T) { }) } } + +func TestPQLPanic(t *testing.T) { + tests := []struct { + name string + call string + }{ + // case 1 + { + name: "StringConditional", + call: "Row(a==foo, a==bar)", + }, + // case 2 + { + name: "StringValue", + call: "Row(a=foo, a=bar)", + }, + // case 3 + { + name: "IntConditional", + call: "Row(a>5, a>6)", + }, + // case 4 + { + name: "IntValue", + call: "Row(a=7, a=8)", + }, + // case 5 + { + name: "List", + call: "Row(a=[7], a=[7,8])", + }, + } + for i, test := range tests { + t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { + var v interface{} + func() { + defer func() { v = recover() }() + + q, err := ParseString(test.call) + if err != nil { + t.Fatalf("parsing query '%s': %v", test.call, err) + } + _ = q + }() + + if !reflect.DeepEqual(v, "multiple instances of argument 'a' provided") { + t.Fatalf("unexpected panic value: %#v", v) + } + + }) + } +} From bdc4f3b07eb7bb4b1eee3a5c148e58fc640e1509 Mon Sep 17 00:00:00 2001 From: Travis Turner Date: Wed, 10 Apr 2019 23:48:17 -0500 Subject: [PATCH 16/16] recover the duplicate arg panic from parser, treat as error --- pql/ast.go | 2 +- pql/parser.go | 20 +++++++++++++++++++- pql/pqlpeg_test.go | 37 ++++++++++++++++++++++--------------- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/pql/ast.go b/pql/ast.go index 36d6e35fb..20b757946 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -119,7 +119,7 @@ func (q *Query) addField(field string) { // key/value. func (q *Query) validateArgField(elem *callStackElem) { if _, exists := elem.call.Args[elem.lastField]; exists { - panic(fmt.Sprintf("multiple instances of argument '%s' provided", elem.lastField)) + panic(fmt.Sprintf("%s: %s", duplicateArgErrorMessage, elem.lastField)) } } diff --git a/pql/parser.go b/pql/parser.go index 611294971..6a28f560e 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -15,6 +15,7 @@ package pql import ( + "fmt" "io" "io/ioutil" "strings" @@ -25,6 +26,9 @@ import ( // timeFormat is the go-style time format used to parse string dates. const timeFormat = "2006-01-02T15:04" +// duplicateArgErrorMessage is used as an error string in the parser. +const duplicateArgErrorMessage = "duplicate argument provided" + // parser represents a parser for the PQL language. type parser struct { r io.Reader @@ -59,6 +63,20 @@ func (p *parser) Parse() (*Query, error) { if err != nil { return nil, errors.Wrap(err, "parsing") } - p.Execute() + + // Handle specific panics from the parser and return them as errors. + var v interface{} + func() { + defer func() { v = recover() }() + p.Execute() + }() + if v != nil { + if strings.HasPrefix(v.(string), duplicateArgErrorMessage) { + return nil, fmt.Errorf("%s", v) + } else { + panic(v) + } + } + return &p.Query, nil } diff --git a/pql/pqlpeg_test.go b/pql/pqlpeg_test.go index f8a6597b5..a90d58a5d 100644 --- a/pql/pqlpeg_test.go +++ b/pql/pqlpeg_test.go @@ -1,6 +1,21 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pql import ( + "fmt" "reflect" "strconv" "testing" @@ -677,7 +692,7 @@ func TestPQLDeepEquality(t *testing.T) { } } -func TestPQLPanic(t *testing.T) { +func TestDuplicateArgError(t *testing.T) { tests := []struct { name string call string @@ -710,21 +725,13 @@ func TestPQLPanic(t *testing.T) { } for i, test := range tests { t.Run(test.name+strconv.Itoa(i), func(t *testing.T) { - var v interface{} - func() { - defer func() { v = recover() }() - - q, err := ParseString(test.call) - if err != nil { - t.Fatalf("parsing query '%s': %v", test.call, err) - } - _ = q - }() - - if !reflect.DeepEqual(v, "multiple instances of argument 'a' provided") { - t.Fatalf("unexpected panic value: %#v", v) + _, err := ParseString(test.call) + expErr := fmt.Sprintf("%s: a", duplicateArgErrorMessage) + if err == nil { + t.Fatalf("expected error for duplicate argument: %s", test.call) + } else if err.Error() != expErr { + t.Fatalf("expected error: %s, but got: %v", expErr, err.Error()) } - }) } }