mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-09-07 00:55:55 +00:00
remove all oldpql and fuzzer code
This commit is contained in:
Matt Jaffee
parent
406f008f80
commit
d1de586ea0
87 changed files with 0 additions and 1575 deletions
|
|
@ -1,8 +0,0 @@
|
|||
See https://github.com/dvyukov/go-fuzz
|
||||
|
||||
|
||||
Quickstart:
|
||||
|
||||
go get -u github.com/dvyukov/go-fuzz/...
|
||||
go-fuzz-build github.com/pilosa/pilosa/pql
|
||||
go-fuzz -bin=./pql-fuzz.zip -workdir=$GOPATH/src/github.com/pilosa/pilosa/pql/fuzz
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(rT03
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(d=f2002-01-01T03:00
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(other!=-2)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Bitmap()
|
||||
|
|
@ -1 +0,0 @@
|
|||
Bitmap(row=10, field=f)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Difference(Bitmap(row=10), Bitmap(row=11))
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(foo<0)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Difference()
|
||||
|
|
@ -1 +0,0 @@
|
|||
Intersect(Bitmap(row=10), Bitmap(row=11))
|
||||
|
|
@ -1 +0,0 @@
|
|||
SV(invalid_column_name=10,f=100)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Intersect()
|
||||
|
|
@ -1 +0,0 @@
|
|||
Union(Bitmap(row=10), Bitmap(row=11))
|
||||
|
|
@ -1 +0,0 @@
|
|||
Union()
|
||||
|
|
@ -1 +0,0 @@
|
|||
Xor(Bitmap(row=10), Bitmap(row=11))
|
||||
|
|
@ -1 +0,0 @@
|
|||
Count(Bitmap(row=10, field=f))
|
||||
|
|
@ -1 +0,0 @@
|
|||
SetBit(row=11, field=f, col=1)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Union( Bitmap() , Count() )
|
||||
|
|
@ -1 +0,0 @@
|
|||
SetValue(col=10, f=25)
|
||||
|
|
@ -1 +0,0 @@
|
|||
SetValue(invalid_column_name=10, f=100)
|
||||
|
|
@ -1 +0,0 @@
|
|||
SetRowAttrs(row=10, field=f, baz=123, bat=true)
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
SetBit(field=f, row=1, col=2, timestamp="1999-12-31T00:00")
|
||||
SetBit(field=f, row=1, col=7, timestamp="2002-01-01T02:00")
|
||||
|
||||
SetBit(field=f, row=1, col=2, timestamp="1999-12-30T00:00")
|
||||
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(row=1, field=f, start="1999-12-31T00:00", end="2002-01-01T03:00")
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
|
||||
Range(foo == 20)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(other != null)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(foo != 20)
|
||||
|
|
@ -1 +0,0 @@
|
|||
N(p(d=0,l=other), d=f,n=3)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(other != -20)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(foo < 20)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Count( Bitmap( id=100))
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(foo <= 20)
|
||||
|
|
@ -1 +0,0 @@
|
|||
SetRowAttrs(row=10, field=f, baz=12.3, bat=.21, bak=-.27, zaz=-0.27 , q=0, zoo="0", do='0')
|
||||
|
|
@ -1 +0,0 @@
|
|||
t( p( ))
|
||||
|
|
@ -1 +0,0 @@
|
|||
SetRowAttrs(row=10,field=f,baz=123,bat=true)
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
|
||||
e(o == 0)
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(r!=-2)
|
||||
|
|
@ -1 +0,0 @@
|
|||
MyCall( y=-12.25, o= -13)
|
||||
|
|
@ -1 +0,0 @@
|
|||
MyCall( key= value, foo="bar", age = 12 , bool0=true, bool1=false, x=null )
|
||||
|
|
@ -1 +0,0 @@
|
|||
t(p(w=1,l=f))
|
||||
|
|
@ -1 +0,0 @@
|
|||
MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(row=1,field=f,start="1999-12-31T00:00",end="2002-01-01T03:00")
|
||||
|
|
@ -1 +0,0 @@
|
|||
MyCall(ke=foo, x =5, y >= 100, z >< [4,8], m != null)
|
||||
|
|
@ -1 +0,0 @@
|
|||
SetValue(invalid_column_name=10,f=100)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(other!=null)
|
||||
|
|
@ -1 +0,0 @@
|
|||
MyCall( key=-12.25, foo= -13)
|
||||
|
|
@ -1 +0,0 @@
|
|||
tRowAttrs(row=1, field=f, baz=13,bat=true)
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(w=:
|
||||
|
|
@ -1 +0,0 @@
|
|||
TopN(field="f", ids=[0,10,30])
|
||||
|
|
@ -1 +0,0 @@
|
|||
n(p() , C
|
||||
|
|
@ -1 +0,0 @@
|
|||
t(p(d=100))
|
||||
|
|
@ -1 +0,0 @@
|
|||
U(B(,C
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(w=12002
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(o <= 0)
|
||||
|
|
@ -1 +0,0 @@
|
|||
t(p(w=0), p(w=1))
|
||||
|
|
@ -1 +0,0 @@
|
|||
n(p(),C(
|
||||
|
|
@ -1 +0,0 @@
|
|||
TopN(Bitmap(id=100, field=other), field=f, n=3)
|
||||
|
|
@ -1 +0,0 @@
|
|||
C(y=12.25,o=13.167,r=2.,z=0.9)
|
||||
|
|
@ -1 +0,0 @@
|
|||
MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)
|
||||
|
|
@ -1 +0,0 @@
|
|||
t(p( ))
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(invalid_column_name<0,f=0)
|
||||
|
|
@ -1 +0,0 @@
|
|||
tV(f=5)
|
||||
|
|
@ -1 +0,0 @@
|
|||
t(Ba(w=0,d=f))
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(o < 0)
|
||||
|
|
@ -1 +0,0 @@
|
|||
n()
|
||||
|
|
@ -1 +0,0 @@
|
|||
Intersect(Bitmap(row=10),Bitmap(row=11))
|
||||
|
|
@ -1 +0,0 @@
|
|||
Cl( k=-12.25, f= -13)
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(o<0)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Difference(Bitmap(row=10),Bitmap(row=11))
|
||||
|
|
@ -1 +0,0 @@
|
|||
j(w=10375035658,t=R)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(other!=l)
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(o<=0)
|
||||
|
|
@ -1 +0,0 @@
|
|||
e(w=T
|
||||
|
|
@ -1 +0,0 @@
|
|||
l(key=oo, x == 12.25, y >= 100, z >< [4,8], m != null)
|
||||
|
|
@ -1 +0,0 @@
|
|||
SB(ow=1, f=f, c=1)
|
||||
|
|
@ -1 +0,0 @@
|
|||
Setalue(invalidcolumnnamf=0)
|
||||
|
|
@ -1 +0,0 @@
|
|||
N(field="f",ids=[0,10,30])
|
||||
|
|
@ -1 +0,0 @@
|
|||
U( B() , C() )
|
||||
|
|
@ -1 +0,0 @@
|
|||
Range(r!=null)
|
||||
|
|
@ -1,272 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package oldpql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Query represents a PQL query.
|
||||
type Query struct {
|
||||
Calls []*Call
|
||||
}
|
||||
|
||||
// WriteCallN returns the number of mutating calls.
|
||||
func (q *Query) WriteCallN() int {
|
||||
var n int
|
||||
for _, call := range q.Calls {
|
||||
switch call.Name {
|
||||
case "SetBit", "ClearBit", "SetRowAttrs", "SetColumnAttrs":
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// String returns a string representation of the query.
|
||||
func (q *Query) String() string {
|
||||
a := make([]string, len(q.Calls))
|
||||
for i, call := range q.Calls {
|
||||
a[i] = call.String()
|
||||
}
|
||||
return strings.Join(a, "\n")
|
||||
}
|
||||
|
||||
// Call represents a function call in the AST.
|
||||
type Call struct {
|
||||
Name string
|
||||
Args map[string]interface{}
|
||||
Children []*Call
|
||||
}
|
||||
|
||||
// UintArg is for reading the value at key from call.Args as a uint64. If the
|
||||
// key is not in Call.Args, the value of the returned bool will be false, and
|
||||
// the error will be nil. The value is assumed to be a uint64 or an int64 and
|
||||
// then cast to a uint64. An error is returned if the value is not an int64 or
|
||||
// uint64.
|
||||
func (c *Call) UintArg(key string) (uint64, bool, error) {
|
||||
val, ok := c.Args[key]
|
||||
if !ok {
|
||||
return 0, false, nil
|
||||
}
|
||||
switch tval := val.(type) {
|
||||
case int64:
|
||||
return uint64(tval), true, nil
|
||||
case uint64:
|
||||
return tval, true, nil
|
||||
default:
|
||||
return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.UintArg", tval, tval)
|
||||
}
|
||||
}
|
||||
|
||||
// UintSliceArg reads the value at key from call.Args as a slice of uint64. If
|
||||
// the key is not in Call.Args, the value of the returned bool will be false,
|
||||
// and the error will be nil. If the value is a slice of int64 it will convert
|
||||
// it to []uint64. Otherwise, if it is not a []uint64 it will return an error.
|
||||
func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) {
|
||||
val, ok := c.Args[key]
|
||||
if !ok {
|
||||
return nil, false, nil
|
||||
}
|
||||
|
||||
switch tval := val.(type) {
|
||||
case []uint64:
|
||||
return tval, true, nil
|
||||
case []int64:
|
||||
ret := make([]uint64, len(tval))
|
||||
for i, v := range tval {
|
||||
ret[i] = uint64(v)
|
||||
}
|
||||
return ret, true, nil
|
||||
default:
|
||||
return nil, true, fmt.Errorf("unexpected type %T in UintSliceArg, val %v", tval, tval)
|
||||
}
|
||||
}
|
||||
|
||||
// Keys returns a list of argument keys in sorted order.
|
||||
func (c *Call) Keys() []string {
|
||||
a := make([]string, 0, len(c.Args))
|
||||
for k := range c.Args {
|
||||
a = append(a, k)
|
||||
}
|
||||
sort.Strings(a)
|
||||
return a
|
||||
}
|
||||
|
||||
// Clone returns a copy of c.
|
||||
func (c *Call) Clone() *Call {
|
||||
if c == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
other := &Call{
|
||||
Name: c.Name,
|
||||
Args: CopyArgs(c.Args),
|
||||
}
|
||||
if c.Children != nil {
|
||||
other.Children = make([]*Call, len(c.Children))
|
||||
for i := range c.Children {
|
||||
other.Children[i] = c.Children[i].Clone()
|
||||
}
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
||||
// String returns the string representation of the call.
|
||||
func (c *Call) String() string {
|
||||
var buf bytes.Buffer
|
||||
|
||||
// Write name.
|
||||
if c.Name != "" {
|
||||
buf.WriteString(c.Name)
|
||||
} else {
|
||||
buf.WriteString("!UNNAMED")
|
||||
}
|
||||
|
||||
// Write opening.
|
||||
buf.WriteByte('(')
|
||||
|
||||
// Write child list.
|
||||
for i, child := range c.Children {
|
||||
if i > 0 {
|
||||
buf.WriteString(", ")
|
||||
}
|
||||
buf.WriteString(child.String())
|
||||
}
|
||||
|
||||
// Separate children and args, if necessary.
|
||||
if len(c.Children) > 0 && len(c.Args) > 0 {
|
||||
buf.WriteString(", ")
|
||||
}
|
||||
|
||||
// Write arguments in key order.
|
||||
for i, key := range c.Keys() {
|
||||
if i > 0 {
|
||||
buf.WriteString(", ")
|
||||
}
|
||||
// If the Arg value is a Condition, then don't include
|
||||
// the equal sign in the string representation.
|
||||
switch v := c.Args[key].(type) {
|
||||
case *Condition:
|
||||
fmt.Fprintf(&buf, "%v %s", key, v.String())
|
||||
default:
|
||||
fmt.Fprintf(&buf, "%v=%s", key, FormatValue(v))
|
||||
}
|
||||
}
|
||||
|
||||
// Write closing.
|
||||
buf.WriteByte(')')
|
||||
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
// HasConditionArg returns true if any arg is a conditional.
|
||||
func (c *Call) HasConditionArg() bool {
|
||||
for _, v := range c.Args {
|
||||
if _, ok := v.(*Condition); ok {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Condition represents an operation & value.
|
||||
// When used in an argument map it represents a binary expression.
|
||||
type Condition struct {
|
||||
Op Token
|
||||
Value interface{}
|
||||
}
|
||||
|
||||
// String returns the string representation of the condition.
|
||||
func (cond *Condition) String() string {
|
||||
return fmt.Sprintf("%s %s", cond.Op.String(), FormatValue(cond.Value))
|
||||
}
|
||||
|
||||
// IntSliceValue reads cond.Value as a slice of uint64.
|
||||
// If the value is a slice of uint64 it will convert
|
||||
// it to []int64. Otherwise, if it is not a []int64 it will return an error.
|
||||
func (cond *Condition) IntSliceValue() ([]int64, error) {
|
||||
val := cond.Value
|
||||
|
||||
switch tval := val.(type) {
|
||||
case []interface{}:
|
||||
ret := make([]int64, len(tval))
|
||||
for i, v := range tval {
|
||||
switch tv := v.(type) {
|
||||
case int64:
|
||||
ret[i] = tv
|
||||
case uint64:
|
||||
ret[i] = int64(tv)
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected value type %T in IntSliceValue, val %v", tv, tv)
|
||||
}
|
||||
}
|
||||
return ret, nil
|
||||
default:
|
||||
return nil, fmt.Errorf("unexpected type %T in IntSliceValue, val %v", tval, tval)
|
||||
}
|
||||
}
|
||||
|
||||
func FormatValue(v interface{}) string {
|
||||
switch v := v.(type) {
|
||||
case string:
|
||||
return fmt.Sprintf("%q", v)
|
||||
case []interface{}:
|
||||
return fmt.Sprintf("%s", joinInterfaceSlice(v))
|
||||
case []uint64:
|
||||
return fmt.Sprintf("%s", joinUint64Slice(v))
|
||||
case time.Time:
|
||||
return fmt.Sprintf("\"%s\"", v.Format(TimeFormat))
|
||||
case *Condition:
|
||||
return v.String()
|
||||
default:
|
||||
return fmt.Sprintf("%v", v)
|
||||
}
|
||||
}
|
||||
|
||||
// CopyArgs returns a copy of m.
|
||||
func CopyArgs(m map[string]interface{}) map[string]interface{} {
|
||||
other := make(map[string]interface{}, len(m))
|
||||
for k, v := range m {
|
||||
other[k] = v
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
||||
func joinInterfaceSlice(a []interface{}) string {
|
||||
other := make([]string, len(a))
|
||||
for i := range a {
|
||||
switch v := a[i].(type) {
|
||||
case string:
|
||||
other[i] = fmt.Sprintf("%q", v)
|
||||
default:
|
||||
other[i] = fmt.Sprintf("%v", v)
|
||||
}
|
||||
}
|
||||
return "[" + strings.Join(other, ",") + "]"
|
||||
}
|
||||
|
||||
func joinUint64Slice(a []uint64) string {
|
||||
other := make([]string, len(a))
|
||||
for i := range a {
|
||||
other[i] = strconv.FormatUint(a[i], 10)
|
||||
}
|
||||
return "[" + strings.Join(other, ",") + "]"
|
||||
}
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package oldpql_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
pql "github.com/pilosa/pilosa/pql/internal/oldpql"
|
||||
)
|
||||
|
||||
// Ensure call can be converted into a string.
|
||||
func TestCall_String(t *testing.T) {
|
||||
t.Run("Empty", func(t *testing.T) {
|
||||
c := &pql.Call{Name: "Bitmap"}
|
||||
if s := c.String(); s != `Bitmap()` {
|
||||
t.Fatalf("unexpected string: %s", s)
|
||||
}
|
||||
})
|
||||
t.Run("With Args", func(t *testing.T) {
|
||||
c := &pql.Call{
|
||||
Name: "Range",
|
||||
Args: map[string]interface{}{
|
||||
"other": "f",
|
||||
"field0": &pql.Condition{Op: pql.GTE, Value: 10},
|
||||
},
|
||||
}
|
||||
if s := c.String(); s != `Range(field0 >= 10, other="f")` {
|
||||
t.Fatalf("unexpected string: %s", s)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// Ensure condition can handle values for BETWEEN operator.
|
||||
func TestCondition_Value(t *testing.T) {
|
||||
t.Run("Between Values", func(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
val []interface{}
|
||||
exp []int64
|
||||
}{
|
||||
{[]interface{}{int64(4), int64(8)}, []int64{4, 8}},
|
||||
{[]interface{}{uint64(4), uint64(8)}, []int64{4, 8}},
|
||||
{[]interface{}{uint64(1), uint64(2), uint64(3)}, []int64{1, 2, 3}},
|
||||
} {
|
||||
c := &pql.Condition{
|
||||
Op: pql.BETWEEN,
|
||||
Value: tt.val,
|
||||
}
|
||||
v, err := c.IntSliceValue()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(v, tt.exp) {
|
||||
t.Fatalf("invalid between values. expected: %v, got %v", tt.exp, v)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
/*
|
||||
package oldpql defines the Pilosa Query Language.
|
||||
*/
|
||||
package oldpql
|
||||
|
|
@ -1,329 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package oldpql
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// TimeFormat is the go-style time format used to parse string dates.
|
||||
const TimeFormat = "2006-01-02T15:04"
|
||||
|
||||
// Parser represents a parser for the PQL language.
|
||||
type Parser struct {
|
||||
scanner *bufScanner
|
||||
}
|
||||
|
||||
// NewParser returns a new instance of Parser.
|
||||
func NewParser(r io.Reader) *Parser {
|
||||
return &Parser{
|
||||
scanner: newBufScanner(r),
|
||||
}
|
||||
}
|
||||
|
||||
// ParseString parses s into a query.
|
||||
func ParseString(s string) (*Query, error) {
|
||||
return NewParser(strings.NewReader(s)).Parse()
|
||||
}
|
||||
|
||||
// Parse parses the next node in the query.
|
||||
func (p *Parser) Parse() (*Query, error) {
|
||||
q := &Query{}
|
||||
for {
|
||||
call, err := p.parseCall()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
q.Calls = append(q.Calls, call)
|
||||
}
|
||||
|
||||
// Require at least one call.
|
||||
if len(q.Calls) == 0 {
|
||||
return nil, io.ErrUnexpectedEOF
|
||||
}
|
||||
|
||||
return q, nil
|
||||
}
|
||||
|
||||
// parseCall parses the next function call.
|
||||
func (p *Parser) parseCall() (*Call, error) {
|
||||
var c Call
|
||||
|
||||
// Read call name.
|
||||
tok, pos, lit := p.scanIgnoreWhitespace()
|
||||
if tok == EOF {
|
||||
return nil, io.EOF
|
||||
} else if tok != IDENT {
|
||||
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
|
||||
}
|
||||
c.Name = lit
|
||||
|
||||
// Scan opening parenthesis.
|
||||
if err := p.expect(LPAREN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Parse children first.
|
||||
children, err := p.parseChildren()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Children = children
|
||||
|
||||
// If next token is a closing paren then exit.
|
||||
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
|
||||
return &c, nil
|
||||
} else if tok == IDENT {
|
||||
p.unscan(1)
|
||||
} else if tok != COMMA {
|
||||
return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit)
|
||||
}
|
||||
|
||||
// Parse key/value arguments.
|
||||
args, err := p.parseArgs()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.Args = args
|
||||
|
||||
// Scan closing parenthesis.
|
||||
if err := p.expect(RPAREN); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
// parseChildren parses call children.
|
||||
func (p *Parser) parseChildren() ([]*Call, error) {
|
||||
var offset int
|
||||
var children []*Call
|
||||
for {
|
||||
// Ensure next two tokens are IDENT+LPAREN.
|
||||
if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT {
|
||||
p.unscanIgnoreWhitespace(1 + offset)
|
||||
return children, nil
|
||||
}
|
||||
if tok, _, _ := p.scan(); tok != LPAREN {
|
||||
p.unscanIgnoreWhitespace(2 + offset)
|
||||
return children, nil
|
||||
}
|
||||
|
||||
// Push tokens back on scanner and parse as a call.
|
||||
p.unscan(2)
|
||||
child, err := p.parseCall()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
children = append(children, child)
|
||||
|
||||
// Exit if closing paren.
|
||||
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
|
||||
p.unscan(1)
|
||||
return children, nil
|
||||
} else if tok != COMMA {
|
||||
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
|
||||
}
|
||||
|
||||
// Make sure comma is unscanned.
|
||||
offset = 1
|
||||
}
|
||||
}
|
||||
|
||||
// parseArgs parses key/value arguments.
|
||||
func (p *Parser) parseArgs() (map[string]interface{}, error) {
|
||||
args := make(map[string]interface{})
|
||||
for {
|
||||
// Parse key.
|
||||
tok, pos, lit := p.scanIgnoreWhitespace()
|
||||
if tok == RPAREN {
|
||||
p.unscan(1)
|
||||
return args, nil
|
||||
} else if tok != IDENT {
|
||||
return nil, parseErrorf(pos, "expected argument key, found %q", lit)
|
||||
}
|
||||
key := lit
|
||||
|
||||
// Expect '=' or a comparison next.
|
||||
var op Token
|
||||
switch tok, pos, lit := p.scanIgnoreWhitespace(); tok {
|
||||
case ASSIGN:
|
||||
case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN:
|
||||
op = tok
|
||||
default:
|
||||
return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit)
|
||||
}
|
||||
|
||||
// Parse value.
|
||||
var value interface{}
|
||||
tok, pos, lit = p.scanIgnoreWhitespace()
|
||||
switch tok {
|
||||
case IDENT:
|
||||
if lit == "true" {
|
||||
value = true
|
||||
} else if lit == "false" {
|
||||
value = false
|
||||
} else if lit == "null" {
|
||||
value = nil
|
||||
} else {
|
||||
value = lit
|
||||
}
|
||||
case STRING:
|
||||
value = lit
|
||||
case INTEGER:
|
||||
v, err := strconv.ParseInt(lit, 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = v
|
||||
case FLOAT:
|
||||
v, err := strconv.ParseFloat(lit, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = v
|
||||
case LBRACK:
|
||||
v, err := p.parseList()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
value = v
|
||||
default:
|
||||
return nil, parseErrorf(pos, "invalid argument value: %q", lit)
|
||||
}
|
||||
|
||||
// Ensure key doesn't already exist.
|
||||
if _, ok := args[key]; ok {
|
||||
return nil, parseErrorf(pos, "argument key already used: %s", key)
|
||||
}
|
||||
|
||||
// If op is specified then create a condition.
|
||||
if op != 0 {
|
||||
value = &Condition{Op: op, Value: value}
|
||||
}
|
||||
|
||||
// Add key/value pair to arguments.
|
||||
args[key] = value
|
||||
|
||||
// Exit if closing paren.
|
||||
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
|
||||
p.unscan(1)
|
||||
return args, nil
|
||||
} else if tok != COMMA {
|
||||
return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// parseList parses a list of primitives. This is used by the TopN() filters.
|
||||
func (p *Parser) parseList() ([]interface{}, error) {
|
||||
var values []interface{}
|
||||
for {
|
||||
// Read next value.
|
||||
tok, pos, lit := p.scanIgnoreWhitespace()
|
||||
switch tok {
|
||||
case IDENT:
|
||||
if lit == "true" {
|
||||
values = append(values, true)
|
||||
} else if lit == "false" {
|
||||
values = append(values, false)
|
||||
} else {
|
||||
values = append(values, lit)
|
||||
}
|
||||
case STRING:
|
||||
values = append(values, lit)
|
||||
case INTEGER:
|
||||
v, err := strconv.ParseInt(lit, 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values = append(values, v)
|
||||
default:
|
||||
return nil, parseErrorf(pos, "invalid list value: %q", lit)
|
||||
}
|
||||
|
||||
// Expect a comma or closing bracket next.
|
||||
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK {
|
||||
break
|
||||
} else if tok != COMMA {
|
||||
return nil, parseErrorf(pos, "expected comma, found %q", lit)
|
||||
}
|
||||
}
|
||||
return values, nil
|
||||
}
|
||||
|
||||
// scan returns the next token from the scanner.
|
||||
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() }
|
||||
|
||||
// scanIgnoreWhitespace returns the next non-whitespace token from the scanner.
|
||||
func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) {
|
||||
tok, pos, lit = p.scan()
|
||||
if tok == WS {
|
||||
tok, pos, lit = p.scan()
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// unscan returns the last n tokens back to the scanner.
|
||||
func (p *Parser) unscan(n int) {
|
||||
for i := 0; i < n; i++ {
|
||||
p.scanner.unscan()
|
||||
}
|
||||
}
|
||||
|
||||
// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner.
|
||||
func (p *Parser) unscanIgnoreWhitespace(n int) {
|
||||
for i := 0; i < n; {
|
||||
p.scanner.unscan()
|
||||
if tok, _, _ := p.scanner.curr(); tok != WS {
|
||||
i++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// expect returns an error if the next token is not exp.
|
||||
func (p *Parser) expect(exp Token) error {
|
||||
if tok, pos, lit := p.scan(); tok != exp {
|
||||
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// pos returns the current position.
|
||||
func (p *Parser) pos() Pos { return p.scanner.pos() }
|
||||
|
||||
// ParseError represents an error that occurred while parsing a PQL query.
|
||||
type ParseError struct {
|
||||
Message string
|
||||
Pos Pos
|
||||
}
|
||||
|
||||
// Error returns a string representation of e.
|
||||
func (e *ParseError) Error() string {
|
||||
return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1)
|
||||
}
|
||||
|
||||
// parseErrorf returns a formatted parse error.
|
||||
func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError {
|
||||
return &ParseError{
|
||||
Message: fmt.Sprintf(format, args...),
|
||||
Pos: pos,
|
||||
}
|
||||
}
|
||||
|
|
@ -1,193 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package oldpql_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
pql "github.com/pilosa/pilosa/pql/internal/oldpql"
|
||||
_ "github.com/pilosa/pilosa/test"
|
||||
)
|
||||
|
||||
// Ensure the parser can parse PQL.
|
||||
func TestParser_Parse(t *testing.T) {
|
||||
// Parse with no children or arguments.
|
||||
t.Run("Empty", func(t *testing.T) {
|
||||
q, err := pql.ParseString(`Bitmap()`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "Bitmap",
|
||||
},
|
||||
) {
|
||||
t.Fatalf("unexpected call: %s", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
// Parse with only children.
|
||||
t.Run("ChildrenOnly", func(t *testing.T) {
|
||||
q, err := pql.ParseString(`Union( Bitmap() , Count() )`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "Union",
|
||||
Children: []*pql.Call{
|
||||
&pql.Call{Name: "Bitmap"},
|
||||
&pql.Call{Name: "Count"},
|
||||
},
|
||||
},
|
||||
) {
|
||||
t.Fatalf("unexpected call: %s", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
// Parse a single child with a single argument.
|
||||
t.Run("ChildWithArgument", func(t *testing.T) {
|
||||
q, err := pql.ParseString(`Count( Bitmap( id=100))`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "Count",
|
||||
Children: []*pql.Call{
|
||||
{Name: "Bitmap", Args: map[string]interface{}{"id": int64(100)}},
|
||||
},
|
||||
},
|
||||
) {
|
||||
t.Fatalf("unexpected call: %s", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
// Parse with only arguments.
|
||||
t.Run("ArgumentsOnly", func(t *testing.T) {
|
||||
q, err := pql.ParseString(`MyCall( key= value, foo="bar", age = 12 , bool0=true, bool1=false, x=null )`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "MyCall",
|
||||
Args: map[string]interface{}{
|
||||
"key": "value",
|
||||
"foo": "bar",
|
||||
"age": int64(12),
|
||||
"bool0": true,
|
||||
"bool1": false,
|
||||
"x": nil,
|
||||
},
|
||||
},
|
||||
) {
|
||||
t.Fatalf("unexpected call: %#v", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
// Parse with float arguments.
|
||||
t.Run("WithFloatArgs", func(t *testing.T) {
|
||||
q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "MyCall",
|
||||
Args: map[string]interface{}{
|
||||
"key": 12.25,
|
||||
"foo": 13.167,
|
||||
"bar": 2.,
|
||||
"baz": 0.9,
|
||||
},
|
||||
},
|
||||
) {
|
||||
t.Fatalf("unexpected call: %#v", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
// Parse with float arguments.
|
||||
t.Run("WithNegativeArgs", func(t *testing.T) {
|
||||
q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "MyCall",
|
||||
Args: map[string]interface{}{
|
||||
"key": -12.25,
|
||||
"foo": int64(-13),
|
||||
},
|
||||
},
|
||||
) {
|
||||
t.Fatalf("unexpected call: %#v", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
// Parse with both child calls and arguments.
|
||||
t.Run("ChildrenAndArguments", func(t *testing.T) {
|
||||
q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "TopN",
|
||||
Children: []*pql.Call{{
|
||||
Name: "Bitmap",
|
||||
Args: map[string]interface{}{"id": int64(100), "field": "other"},
|
||||
}},
|
||||
Args: map[string]interface{}{"n": int64(3), "field": "f"},
|
||||
},
|
||||
) {
|
||||
t.Fatalf("unexpected call: %#v", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
// Parse a list argument.
|
||||
t.Run("ListArgument", func(t *testing.T) {
|
||||
q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "TopN",
|
||||
Args: map[string]interface{}{
|
||||
"field": "f",
|
||||
"ids": []interface{}{int64(0), int64(10), int64(30)},
|
||||
},
|
||||
},
|
||||
) {
|
||||
t.Fatalf("unexpected call: %#v", q.Calls[0])
|
||||
}
|
||||
})
|
||||
|
||||
// Parse with condition arguments.
|
||||
t.Run("WithCondition", func(t *testing.T) {
|
||||
q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(q.Calls[0],
|
||||
&pql.Call{
|
||||
Name: "MyCall",
|
||||
Args: map[string]interface{}{
|
||||
"key": "foo",
|
||||
"x": &pql.Condition{Op: pql.EQ, Value: 12.25},
|
||||
"y": &pql.Condition{Op: pql.GTE, Value: int64(100)},
|
||||
"z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}},
|
||||
"m": &pql.Condition{Op: pql.NEQ, Value: nil},
|
||||
},
|
||||
},
|
||||
) {
|
||||
t.Fatalf("unexpected call: %#v", q.Calls[0])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
|
@ -1,303 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package oldpql
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"io"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// Scanner represents a PQL lexical scanner.
|
||||
type Scanner struct {
|
||||
r io.RuneScanner
|
||||
pos Pos
|
||||
}
|
||||
|
||||
// NewScanner returns a new instance of Scanner.
|
||||
func NewScanner(r io.Reader) *Scanner {
|
||||
return &Scanner{r: bufio.NewReader(r)}
|
||||
}
|
||||
|
||||
// Scan returns the next token and position from the underlying reader.
|
||||
func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
|
||||
// Read next code point.
|
||||
ch := s.read()
|
||||
|
||||
// If we see whitespace then consume all contiguous whitespace.
|
||||
// If we see a letter, or certain acceptable special characters, then consume
|
||||
// as an ident or reserved word. If we see quotes, then scan as string.
|
||||
if isWhitespace(ch) {
|
||||
s.unread()
|
||||
return s.scanWhitespace()
|
||||
} else if isIdentFirstChar(ch) {
|
||||
s.unread()
|
||||
return s.scanIdent()
|
||||
} else if isDigit(ch) || ch == '-' {
|
||||
s.unread()
|
||||
return s.scanNumber()
|
||||
} else if ch == '"' || ch == '\'' {
|
||||
s.unread()
|
||||
return s.scanString()
|
||||
}
|
||||
|
||||
// Otherwise parse individual characters.
|
||||
switch ch {
|
||||
case eof:
|
||||
return EOF, pos, ""
|
||||
case '=':
|
||||
if next := s.read(); next == '=' {
|
||||
return EQ, pos, "=="
|
||||
}
|
||||
s.unread()
|
||||
return ASSIGN, pos, string(ch)
|
||||
case '!':
|
||||
if next := s.read(); next == '=' {
|
||||
return NEQ, pos, "!="
|
||||
}
|
||||
s.unread()
|
||||
return ILLEGAL, pos, string(ch)
|
||||
case '<':
|
||||
if next := s.read(); next == '=' {
|
||||
return LTE, pos, "<="
|
||||
}
|
||||
s.unread()
|
||||
return LT, pos, string(ch)
|
||||
case '>':
|
||||
next := s.read()
|
||||
if next == '=' {
|
||||
return GTE, pos, ">="
|
||||
} else if next == '<' {
|
||||
return BETWEEN, pos, "><"
|
||||
}
|
||||
s.unread()
|
||||
return GT, pos, string(ch)
|
||||
case ',':
|
||||
return COMMA, pos, string(ch)
|
||||
case '(':
|
||||
return LPAREN, pos, string(ch)
|
||||
case ')':
|
||||
return RPAREN, pos, string(ch)
|
||||
case '[':
|
||||
return LBRACK, pos, string(ch)
|
||||
case ']':
|
||||
return RBRACK, pos, string(ch)
|
||||
default:
|
||||
return ILLEGAL, pos, string(ch)
|
||||
}
|
||||
}
|
||||
|
||||
// read returns the next code point from the underlying reader and updates the pos.
|
||||
func (s *Scanner) read() rune {
|
||||
// Read next rune from underlying reader.
|
||||
ch, _, err := s.r.ReadRune()
|
||||
if err != nil {
|
||||
return eof
|
||||
}
|
||||
|
||||
// Update position information.
|
||||
if ch == '\n' {
|
||||
s.pos.Line++
|
||||
s.pos.Char = 0
|
||||
} else {
|
||||
s.pos.Char++
|
||||
}
|
||||
|
||||
return ch
|
||||
}
|
||||
|
||||
// unread pushes the previously read rune back onto the reader.
|
||||
func (s *Scanner) unread() {
|
||||
if s.pos.Char == 0 {
|
||||
s.pos.Line--
|
||||
} else {
|
||||
s.pos.Char--
|
||||
}
|
||||
|
||||
s.r.UnreadRune()
|
||||
}
|
||||
|
||||
// scanWhitespace consumes the current rune and all contiguous whitespace.
|
||||
func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
|
||||
var buf bytes.Buffer
|
||||
for {
|
||||
ch := s.read()
|
||||
if ch == eof {
|
||||
break
|
||||
} else if !isWhitespace(ch) {
|
||||
s.unread()
|
||||
break
|
||||
}
|
||||
buf.WriteRune(ch)
|
||||
}
|
||||
|
||||
return WS, pos, buf.String()
|
||||
}
|
||||
|
||||
func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
|
||||
var buf bytes.Buffer
|
||||
for {
|
||||
ch := s.read()
|
||||
if ch == eof {
|
||||
break
|
||||
} else if !isIdentChar(ch) {
|
||||
s.unread()
|
||||
break
|
||||
}
|
||||
buf.WriteRune(ch)
|
||||
}
|
||||
lit = buf.String()
|
||||
|
||||
// If the literal matches a keyword then return that keyword.
|
||||
if tok = Lookup(lit); tok != IDENT {
|
||||
return tok, pos, lit
|
||||
}
|
||||
|
||||
return IDENT, pos, lit
|
||||
}
|
||||
|
||||
// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character.
|
||||
func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
tok = INTEGER
|
||||
|
||||
var buf bytes.Buffer
|
||||
var seenDot bool
|
||||
first := true
|
||||
for {
|
||||
ch := s.read()
|
||||
if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') {
|
||||
s.unread()
|
||||
break
|
||||
}
|
||||
if ch == '.' {
|
||||
seenDot = true
|
||||
tok = FLOAT
|
||||
}
|
||||
buf.WriteRune(ch)
|
||||
first = false
|
||||
}
|
||||
return tok, pos, buf.String()
|
||||
}
|
||||
|
||||
// scanString consumes a single-quoted or double-quoted string.
|
||||
func (s *Scanner) scanString() (tok Token, pos Pos, lit string) {
|
||||
pos = s.pos
|
||||
|
||||
// This must be either a single- or double-quote.
|
||||
ending := s.read()
|
||||
|
||||
var buf bytes.Buffer
|
||||
for {
|
||||
ch := s.read()
|
||||
if ch == ending {
|
||||
break
|
||||
} else if ch == '\n' || ch == eof {
|
||||
return BADSTRING, pos, buf.String()
|
||||
} else if ch == '\\' {
|
||||
next := s.read()
|
||||
if next == 'n' {
|
||||
buf.WriteRune('\n')
|
||||
} else if next == '\\' {
|
||||
buf.WriteRune('\\')
|
||||
} else if next == '"' {
|
||||
buf.WriteRune('"')
|
||||
} else if next == '\'' {
|
||||
buf.WriteRune('\'')
|
||||
} else {
|
||||
return BADSTRING, pos, buf.String()
|
||||
}
|
||||
} else {
|
||||
buf.WriteRune(ch)
|
||||
}
|
||||
}
|
||||
|
||||
return STRING, pos, buf.String()
|
||||
}
|
||||
|
||||
// bufScanner represents a wrapper for scanner to add a buffer.
|
||||
// It provides a fixed-length circular buffer that can be unread.
|
||||
type bufScanner struct {
|
||||
s *Scanner
|
||||
i int // buffer index
|
||||
n int // buffer size
|
||||
buf [8]struct {
|
||||
tok Token
|
||||
pos Pos
|
||||
lit string
|
||||
}
|
||||
}
|
||||
|
||||
// newBufScanner returns a new buffered scanner for a reader.
|
||||
func newBufScanner(r io.Reader) *bufScanner {
|
||||
return &bufScanner{s: NewScanner(r)}
|
||||
}
|
||||
|
||||
// Scan reads the next token from the scanner.
|
||||
func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) {
|
||||
// If we have unread tokens then read them off the buffer first.
|
||||
if s.n > 0 {
|
||||
s.n--
|
||||
return s.curr()
|
||||
}
|
||||
|
||||
// Move buffer position forward and save the token.
|
||||
s.i = (s.i + 1) % len(s.buf)
|
||||
buf := &s.buf[s.i]
|
||||
buf.tok, buf.pos, buf.lit = s.s.Scan()
|
||||
|
||||
return s.curr()
|
||||
}
|
||||
|
||||
// unscan pushes the previously token back onto the buffer.
|
||||
func (s *bufScanner) unscan() { s.n++ }
|
||||
|
||||
// curr returns the last read token.
|
||||
func (s *bufScanner) curr() (tok Token, pos Pos, lit string) {
|
||||
buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)]
|
||||
return buf.tok, buf.pos, buf.lit
|
||||
}
|
||||
|
||||
// pos returns the current position.
|
||||
func (s *bufScanner) pos() Pos {
|
||||
_, pos, _ := s.curr()
|
||||
return pos
|
||||
}
|
||||
|
||||
// isWhitespace returns true if the rune a Unicode space character.
|
||||
func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) }
|
||||
|
||||
// isLetter returns true if the rune is a letter.
|
||||
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
|
||||
|
||||
// isDigit returns true if the rune is a digit.
|
||||
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
|
||||
|
||||
// isIdentChar returns true if the rune can be used in an unquoted identifier.
|
||||
func isIdentChar(ch rune) bool {
|
||||
return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.'
|
||||
}
|
||||
|
||||
// isIdentFirstChar returns true if the rune can be used as the first char in an identifier.
|
||||
func isIdentFirstChar(ch rune) bool { return isLetter(ch) }
|
||||
|
||||
const eof = rune(0)
|
||||
|
|
@ -1,74 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package oldpql_test
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
pql "github.com/pilosa/pilosa/pql/internal/oldpql"
|
||||
)
|
||||
|
||||
func TestScanner_Scan(t *testing.T) {
|
||||
var tests = []struct {
|
||||
name string
|
||||
s string
|
||||
tok pql.Token
|
||||
lit string
|
||||
pos pql.Pos
|
||||
}{
|
||||
// Special tokens (EOF, ILLEGAL, WS)
|
||||
{name: "EOF", s: ``, tok: pql.EOF},
|
||||
{name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`},
|
||||
{name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "},
|
||||
{name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"},
|
||||
{name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"},
|
||||
|
||||
{name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`},
|
||||
{name: "EQ", s: `==`, tok: pql.EQ, lit: `==`},
|
||||
{name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`},
|
||||
{name: "LT", s: `<`, tok: pql.LT, lit: `<`},
|
||||
{name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`},
|
||||
{name: "GT", s: `>`, tok: pql.GT, lit: `>`},
|
||||
{name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`},
|
||||
{name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`},
|
||||
{name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`},
|
||||
{name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`},
|
||||
{name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`},
|
||||
{name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`},
|
||||
{name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`},
|
||||
|
||||
{name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`},
|
||||
{name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`},
|
||||
{name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`},
|
||||
|
||||
{name: "ALL", s: `all`, tok: pql.ALL, lit: `all`},
|
||||
{name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive
|
||||
}
|
||||
|
||||
for i, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
s := pql.NewScanner(strings.NewReader(tt.s))
|
||||
tok, pos, lit := s.Scan()
|
||||
if tt.tok != tok {
|
||||
t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit)
|
||||
} else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char {
|
||||
t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos)
|
||||
} else if tt.lit != lit {
|
||||
t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
|
@ -1,111 +0,0 @@
|
|||
// Copyright 2017 Pilosa Corp.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package oldpql
|
||||
|
||||
import "strings"
|
||||
|
||||
// Token is a lexical token of the PQL language.
|
||||
type Token int
|
||||
|
||||
const (
|
||||
// Special tokens
|
||||
ILLEGAL Token = iota
|
||||
EOF
|
||||
WS
|
||||
|
||||
literal_beg
|
||||
IDENT // main
|
||||
STRING // "foo"
|
||||
BADSTRING // bad escape or unclosed string
|
||||
INTEGER // 12345
|
||||
FLOAT // 100.2
|
||||
literal_end
|
||||
|
||||
keyword_beg
|
||||
ALL
|
||||
keyword_end
|
||||
|
||||
ASSIGN // =
|
||||
EQ // ==
|
||||
NEQ // !=
|
||||
LT // <
|
||||
LTE // <=
|
||||
GT // >
|
||||
GTE // >=
|
||||
BETWEEN // ><
|
||||
COMMA // ,
|
||||
LPAREN // (
|
||||
RPAREN // )
|
||||
LBRACK // (
|
||||
RBRACK // )
|
||||
)
|
||||
|
||||
var tokens = [...]string{
|
||||
ILLEGAL: "ILLEGAL",
|
||||
EOF: "EOF",
|
||||
WS: "WS",
|
||||
|
||||
IDENT: "IDENT",
|
||||
INTEGER: "INTEGER",
|
||||
FLOAT: "FLOAT",
|
||||
|
||||
ALL: "ALL",
|
||||
|
||||
ASSIGN: "=",
|
||||
EQ: "==",
|
||||
NEQ: "!=",
|
||||
LT: "<",
|
||||
LTE: "<=",
|
||||
GT: ">",
|
||||
GTE: ">=",
|
||||
BETWEEN: "><",
|
||||
COMMA: ",",
|
||||
LPAREN: "(",
|
||||
RPAREN: ")",
|
||||
LBRACK: "(",
|
||||
RBRACK: ")",
|
||||
}
|
||||
|
||||
var keywords map[string]Token
|
||||
|
||||
func init() {
|
||||
keywords = make(map[string]Token)
|
||||
for tok := keyword_beg + 1; tok < keyword_end; tok++ {
|
||||
keywords[strings.ToLower(tokens[tok])] = tok
|
||||
}
|
||||
}
|
||||
|
||||
// String returns the string representation of the token.
|
||||
func (tok Token) String() string {
|
||||
if tok >= 0 && tok < Token(len(tokens)) {
|
||||
return tokens[tok]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Lookup returns the token associated with a given string.
|
||||
func Lookup(ident string) Token {
|
||||
if tok, ok := keywords[strings.ToLower(ident)]; ok {
|
||||
return tok
|
||||
}
|
||||
return IDENT
|
||||
}
|
||||
|
||||
// Pos specifies the line and character position of a token.
|
||||
// The Char and Line are both zero-based indexes.
|
||||
type Pos struct {
|
||||
Line int
|
||||
Char int
|
||||
}
|
||||
|
|
@ -1,115 +0,0 @@
|
|||
// +build gofuzz
|
||||
|
||||
package pql
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
|
||||
"github.com/pilosa/pilosa/pql/internal/oldpql"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
func Fuzz(data []byte) int {
|
||||
p1 := NewParser(bytes.NewReader(data))
|
||||
q1, err1 := p1.Parse()
|
||||
p2 := oldpql.NewParser(bytes.NewReader(data))
|
||||
q2, err2 := p2.Parse()
|
||||
if err1 != nil && err2 != nil {
|
||||
return 0 // both error - this is fine
|
||||
}
|
||||
if err1 != nil || err2 != nil {
|
||||
// error in one but not both - need to know this
|
||||
panic(fmt.Sprintf("Query: '%s' errored one but not both.\n%v\n%v\n", data, err1, err2))
|
||||
}
|
||||
|
||||
// if parsers got different results
|
||||
if err := queriesEqual(q1, q2); err != nil {
|
||||
panic(fmt.Sprintf(`Query: '%s' parsed, but got different results:
|
||||
Result New (string)
|
||||
%s
|
||||
Result New (hashv)
|
||||
%#v
|
||||
Result Old (string)
|
||||
%s
|
||||
Result Old (hashv)
|
||||
%#v
|
||||
err:
|
||||
%v
|
||||
`, data, q1, q1, q2, q2, err))
|
||||
}
|
||||
|
||||
// both queries parsed succesfully and got equivalent results
|
||||
return 1
|
||||
}
|
||||
|
||||
func queriesEqual(q1 *Query, q2 *oldpql.Query) (err error) {
|
||||
if q1.String() != q2.String() {
|
||||
defer func() {
|
||||
// golang black magic
|
||||
if err == nil {
|
||||
err = errors.New("string reps unequal")
|
||||
} else {
|
||||
err = errors.Wrap(err, "string reps unequal")
|
||||
}
|
||||
}()
|
||||
}
|
||||
if len(q1.Calls) != len(q2.Calls) {
|
||||
return errors.Errorf("call lengths unequal: %d and %d", len(q1.Calls), len(q2.Calls))
|
||||
}
|
||||
for i, c1 := range q1.Calls {
|
||||
c2 := q2.Calls[i]
|
||||
if err := callsEqual(c1, c2); err != nil {
|
||||
return errors.Wrapf(err, "calls at %d not equal", i)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func callsEqual(c1 *Call, c2 *oldpql.Call) error {
|
||||
if err := argsEqual(c1.Args, c2.Args); err != nil {
|
||||
return errors.Wrap(err, "args unequal")
|
||||
}
|
||||
if c1.Name != c2.Name {
|
||||
return errors.Errorf("names unequal '%s' != '%s'", c1.Name, c2.Name)
|
||||
}
|
||||
if len(c1.Children) != len(c2.Children) {
|
||||
return errors.Errorf("different child lengths %d and %d", len(c1.Children), len(c2.Children))
|
||||
}
|
||||
|
||||
for i, child1 := range c1.Children {
|
||||
child2 := c2.Children[i]
|
||||
if err := callsEqual(child1, child2); err != nil {
|
||||
return errors.Wrapf(err, "children at %d not equal", i)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func argsEqual(a1 map[string]interface{}, a2 map[string]interface{}) error {
|
||||
if len(a1) != len(a2) {
|
||||
return errors.Errorf("lengths unequal %d and %d", len(a1), len(a2))
|
||||
}
|
||||
|
||||
for k, v1 := range a1 {
|
||||
v2 := a1[k]
|
||||
if c1, ok := v1.(Condition); ok {
|
||||
if c2, ok := v2.(oldpql.Condition); ok {
|
||||
if int(c1.Op) != int(c2.Op) {
|
||||
return errors.Errorf("condition ops unequal %d %d", c1, c2)
|
||||
}
|
||||
if !reflect.DeepEqual(c1.Value, c2.Value) {
|
||||
return errors.Errorf("condition values unequal '%v' '%v'", c1.Value, c2.Value)
|
||||
}
|
||||
continue
|
||||
}
|
||||
return errors.Errorf("values at %s unequal '%v' '%v'", k, v1, v2)
|
||||
}
|
||||
if !reflect.DeepEqual(v1, v2) {
|
||||
return errors.Errorf("values at %s unequal '%v' '%v'", k, v1, v2)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Add table
Reference in a new issue