diff --git a/pql/internal/oldpql/ast.go b/pql/internal/oldpql/ast.go new file mode 100644 index 000000000..bee778905 --- /dev/null +++ b/pql/internal/oldpql/ast.go @@ -0,0 +1,272 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql + +import ( + "bytes" + "fmt" + "sort" + "strconv" + "strings" + "time" +) + +// Query represents a PQL query. +type Query struct { + Calls []*Call +} + +// WriteCallN returns the number of mutating calls. +func (q *Query) WriteCallN() int { + var n int + for _, call := range q.Calls { + switch call.Name { + case "SetBit", "ClearBit", "SetRowAttrs", "SetColumnAttrs": + n++ + } + } + return n +} + +// String returns a string representation of the query. +func (q *Query) String() string { + a := make([]string, len(q.Calls)) + for i, call := range q.Calls { + a[i] = call.String() + } + return strings.Join(a, "\n") +} + +// Call represents a function call in the AST. +type Call struct { + Name string + Args map[string]interface{} + Children []*Call +} + +// UintArg is for reading the value at key from call.Args as a uint64. If the +// key is not in Call.Args, the value of the returned bool will be false, and +// the error will be nil. The value is assumed to be a uint64 or an int64 and +// then cast to a uint64. An error is returned if the value is not an int64 or +// uint64. +func (c *Call) UintArg(key string) (uint64, bool, error) { + val, ok := c.Args[key] + if !ok { + return 0, false, nil + } + switch tval := val.(type) { + case int64: + return uint64(tval), true, nil + case uint64: + return tval, true, nil + default: + return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.UintArg", tval, tval) + } +} + +// UintSliceArg reads the value at key from call.Args as a slice of uint64. If +// the key is not in Call.Args, the value of the returned bool will be false, +// and the error will be nil. If the value is a slice of int64 it will convert +// it to []uint64. Otherwise, if it is not a []uint64 it will return an error. +func (c *Call) UintSliceArg(key string) ([]uint64, bool, error) { + val, ok := c.Args[key] + if !ok { + return nil, false, nil + } + + switch tval := val.(type) { + case []uint64: + return tval, true, nil + case []int64: + ret := make([]uint64, len(tval)) + for i, v := range tval { + ret[i] = uint64(v) + } + return ret, true, nil + default: + return nil, true, fmt.Errorf("unexpected type %T in UintSliceArg, val %v", tval, tval) + } +} + +// Keys returns a list of argument keys in sorted order. +func (c *Call) Keys() []string { + a := make([]string, 0, len(c.Args)) + for k := range c.Args { + a = append(a, k) + } + sort.Strings(a) + return a +} + +// Clone returns a copy of c. +func (c *Call) Clone() *Call { + if c == nil { + return nil + } + + other := &Call{ + Name: c.Name, + Args: CopyArgs(c.Args), + } + if c.Children != nil { + other.Children = make([]*Call, len(c.Children)) + for i := range c.Children { + other.Children[i] = c.Children[i].Clone() + } + } + return other +} + +// String returns the string representation of the call. +func (c *Call) String() string { + var buf bytes.Buffer + + // Write name. + if c.Name != "" { + buf.WriteString(c.Name) + } else { + buf.WriteString("!UNNAMED") + } + + // Write opening. + buf.WriteByte('(') + + // Write child list. + for i, child := range c.Children { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(child.String()) + } + + // Separate children and args, if necessary. + if len(c.Children) > 0 && len(c.Args) > 0 { + buf.WriteString(", ") + } + + // Write arguments in key order. + for i, key := range c.Keys() { + if i > 0 { + buf.WriteString(", ") + } + // If the Arg value is a Condition, then don't include + // the equal sign in the string representation. + switch v := c.Args[key].(type) { + case *Condition: + fmt.Fprintf(&buf, "%v %s", key, v.String()) + default: + fmt.Fprintf(&buf, "%v=%s", key, FormatValue(v)) + } + } + + // Write closing. + buf.WriteByte(')') + + return buf.String() +} + +// HasConditionArg returns true if any arg is a conditional. +func (c *Call) HasConditionArg() bool { + for _, v := range c.Args { + if _, ok := v.(*Condition); ok { + return true + } + } + return false +} + +// Condition represents an operation & value. +// When used in an argument map it represents a binary expression. +type Condition struct { + Op Token + Value interface{} +} + +// String returns the string representation of the condition. +func (cond *Condition) String() string { + return fmt.Sprintf("%s %s", cond.Op.String(), FormatValue(cond.Value)) +} + +// IntSliceValue reads cond.Value as a slice of uint64. +// If the value is a slice of uint64 it will convert +// it to []int64. Otherwise, if it is not a []int64 it will return an error. +func (cond *Condition) IntSliceValue() ([]int64, error) { + val := cond.Value + + switch tval := val.(type) { + case []interface{}: + ret := make([]int64, len(tval)) + for i, v := range tval { + switch tv := v.(type) { + case int64: + ret[i] = tv + case uint64: + ret[i] = int64(tv) + default: + return nil, fmt.Errorf("unexpected value type %T in IntSliceValue, val %v", tv, tv) + } + } + return ret, nil + default: + return nil, fmt.Errorf("unexpected type %T in IntSliceValue, val %v", tval, tval) + } +} + +func FormatValue(v interface{}) string { + switch v := v.(type) { + case string: + return fmt.Sprintf("%q", v) + case []interface{}: + return fmt.Sprintf("%s", joinInterfaceSlice(v)) + case []uint64: + return fmt.Sprintf("%s", joinUint64Slice(v)) + case time.Time: + return fmt.Sprintf("\"%s\"", v.Format(TimeFormat)) + case *Condition: + return v.String() + default: + return fmt.Sprintf("%v", v) + } +} + +// CopyArgs returns a copy of m. +func CopyArgs(m map[string]interface{}) map[string]interface{} { + other := make(map[string]interface{}, len(m)) + for k, v := range m { + other[k] = v + } + return other +} + +func joinInterfaceSlice(a []interface{}) string { + other := make([]string, len(a)) + for i := range a { + switch v := a[i].(type) { + case string: + other[i] = fmt.Sprintf("%q", v) + default: + other[i] = fmt.Sprintf("%v", v) + } + } + return "[" + strings.Join(other, ",") + "]" +} + +func joinUint64Slice(a []uint64) string { + other := make([]string, len(a)) + for i := range a { + other[i] = strconv.FormatUint(a[i], 10) + } + return "[" + strings.Join(other, ",") + "]" +} diff --git a/pql/internal/oldpql/ast_test.go b/pql/internal/oldpql/ast_test.go new file mode 100644 index 000000000..1b7c9eba0 --- /dev/null +++ b/pql/internal/oldpql/ast_test.go @@ -0,0 +1,69 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql_test + +import ( + "reflect" + "testing" + + pql "github.com/pilosa/pilosa/pql/internal/oldpql" +) + +// Ensure call can be converted into a string. +func TestCall_String(t *testing.T) { + t.Run("Empty", func(t *testing.T) { + c := &pql.Call{Name: "Bitmap"} + if s := c.String(); s != `Bitmap()` { + t.Fatalf("unexpected string: %s", s) + } + }) + t.Run("With Args", func(t *testing.T) { + c := &pql.Call{ + Name: "Range", + Args: map[string]interface{}{ + "other": "f", + "field0": &pql.Condition{Op: pql.GTE, Value: 10}, + }, + } + if s := c.String(); s != `Range(field0 >= 10, other="f")` { + t.Fatalf("unexpected string: %s", s) + } + }) +} + +// Ensure condition can handle values for BETWEEN operator. +func TestCondition_Value(t *testing.T) { + t.Run("Between Values", func(t *testing.T) { + for _, tt := range []struct { + val []interface{} + exp []int64 + }{ + {[]interface{}{int64(4), int64(8)}, []int64{4, 8}}, + {[]interface{}{uint64(4), uint64(8)}, []int64{4, 8}}, + {[]interface{}{uint64(1), uint64(2), uint64(3)}, []int64{1, 2, 3}}, + } { + c := &pql.Condition{ + Op: pql.BETWEEN, + Value: tt.val, + } + v, err := c.IntSliceValue() + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(v, tt.exp) { + t.Fatalf("invalid between values. expected: %v, got %v", tt.exp, v) + } + } + }) +} diff --git a/pql/internal/oldpql/doc.go b/pql/internal/oldpql/doc.go new file mode 100644 index 000000000..3e5bd4876 --- /dev/null +++ b/pql/internal/oldpql/doc.go @@ -0,0 +1,18 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +package oldpql defines the Pilosa Query Language. +*/ +package oldpql diff --git a/pql/internal/oldpql/parser.go b/pql/internal/oldpql/parser.go new file mode 100644 index 000000000..d54033ee7 --- /dev/null +++ b/pql/internal/oldpql/parser.go @@ -0,0 +1,329 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql + +import ( + "fmt" + "io" + "strconv" + "strings" +) + +// TimeFormat is the go-style time format used to parse string dates. +const TimeFormat = "2006-01-02T15:04" + +// Parser represents a parser for the PQL language. +type Parser struct { + scanner *bufScanner +} + +// NewParser returns a new instance of Parser. +func NewParser(r io.Reader) *Parser { + return &Parser{ + scanner: newBufScanner(r), + } +} + +// ParseString parses s into a query. +func ParseString(s string) (*Query, error) { + return NewParser(strings.NewReader(s)).Parse() +} + +// Parse parses the next node in the query. +func (p *Parser) Parse() (*Query, error) { + q := &Query{} + for { + call, err := p.parseCall() + if err == io.EOF { + break + } else if err != nil { + return nil, err + } + q.Calls = append(q.Calls, call) + } + + // Require at least one call. + if len(q.Calls) == 0 { + return nil, io.ErrUnexpectedEOF + } + + return q, nil +} + +// parseCall parses the next function call. +func (p *Parser) parseCall() (*Call, error) { + var c Call + + // Read call name. + tok, pos, lit := p.scanIgnoreWhitespace() + if tok == EOF { + return nil, io.EOF + } else if tok != IDENT { + return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos} + } + c.Name = lit + + // Scan opening parenthesis. + if err := p.expect(LPAREN); err != nil { + return nil, err + } + + // Parse children first. + children, err := p.parseChildren() + if err != nil { + return nil, err + } + c.Children = children + + // If next token is a closing paren then exit. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { + return &c, nil + } else if tok == IDENT { + p.unscan(1) + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma, right paren, or identifier, found %q", lit) + } + + // Parse key/value arguments. + args, err := p.parseArgs() + if err != nil { + return nil, err + } + c.Args = args + + // Scan closing parenthesis. + if err := p.expect(RPAREN); err != nil { + return nil, err + } + + return &c, nil +} + +// parseChildren parses call children. +func (p *Parser) parseChildren() ([]*Call, error) { + var offset int + var children []*Call + for { + // Ensure next two tokens are IDENT+LPAREN. + if tok, _, _ := p.scanIgnoreWhitespace(); tok != IDENT { + p.unscanIgnoreWhitespace(1 + offset) + return children, nil + } + if tok, _, _ := p.scan(); tok != LPAREN { + p.unscanIgnoreWhitespace(2 + offset) + return children, nil + } + + // Push tokens back on scanner and parse as a call. + p.unscan(2) + child, err := p.parseCall() + if err != nil { + return nil, err + } + children = append(children, child) + + // Exit if closing paren. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { + p.unscan(1) + return children, nil + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) + } + + // Make sure comma is unscanned. + offset = 1 + } +} + +// parseArgs parses key/value arguments. +func (p *Parser) parseArgs() (map[string]interface{}, error) { + args := make(map[string]interface{}) + for { + // Parse key. + tok, pos, lit := p.scanIgnoreWhitespace() + if tok == RPAREN { + p.unscan(1) + return args, nil + } else if tok != IDENT { + return nil, parseErrorf(pos, "expected argument key, found %q", lit) + } + key := lit + + // Expect '=' or a comparison next. + var op Token + switch tok, pos, lit := p.scanIgnoreWhitespace(); tok { + case ASSIGN: + case EQ, NEQ, LT, LTE, GT, GTE, BETWEEN: + op = tok + default: + return nil, parseErrorf(pos, "expected equals sign or comparison operator, found %q", lit) + } + + // Parse value. + var value interface{} + tok, pos, lit = p.scanIgnoreWhitespace() + switch tok { + case IDENT: + if lit == "true" { + value = true + } else if lit == "false" { + value = false + } else if lit == "null" { + value = nil + } else { + value = lit + } + case STRING: + value = lit + case INTEGER: + v, err := strconv.ParseInt(lit, 10, 64) + if err != nil { + return nil, err + } + value = v + case FLOAT: + v, err := strconv.ParseFloat(lit, 64) + if err != nil { + return nil, err + } + value = v + case LBRACK: + v, err := p.parseList() + if err != nil { + return nil, err + } + value = v + default: + return nil, parseErrorf(pos, "invalid argument value: %q", lit) + } + + // Ensure key doesn't already exist. + if _, ok := args[key]; ok { + return nil, parseErrorf(pos, "argument key already used: %s", key) + } + + // If op is specified then create a condition. + if op != 0 { + value = &Condition{Op: op, Value: value} + } + + // Add key/value pair to arguments. + args[key] = value + + // Exit if closing paren. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN { + p.unscan(1) + return args, nil + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma or right paren, found %q", lit) + } + } +} + +// parseList parses a list of primitives. This is used by the TopN() filters. +func (p *Parser) parseList() ([]interface{}, error) { + var values []interface{} + for { + // Read next value. + tok, pos, lit := p.scanIgnoreWhitespace() + switch tok { + case IDENT: + if lit == "true" { + values = append(values, true) + } else if lit == "false" { + values = append(values, false) + } else { + values = append(values, lit) + } + case STRING: + values = append(values, lit) + case INTEGER: + v, err := strconv.ParseInt(lit, 10, 64) + if err != nil { + return nil, err + } + values = append(values, v) + default: + return nil, parseErrorf(pos, "invalid list value: %q", lit) + } + + // Expect a comma or closing bracket next. + if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RBRACK { + break + } else if tok != COMMA { + return nil, parseErrorf(pos, "expected comma, found %q", lit) + } + } + return values, nil +} + +// scan returns the next token from the scanner. +func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() } + +// scanIgnoreWhitespace returns the next non-whitespace token from the scanner. +func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) { + tok, pos, lit = p.scan() + if tok == WS { + tok, pos, lit = p.scan() + } + return +} + +// unscan returns the last n tokens back to the scanner. +func (p *Parser) unscan(n int) { + for i := 0; i < n; i++ { + p.scanner.unscan() + } +} + +// unscanIgnoreWhitespace returns the last n non-WS tokens back to the scanner. +func (p *Parser) unscanIgnoreWhitespace(n int) { + for i := 0; i < n; { + p.scanner.unscan() + if tok, _, _ := p.scanner.curr(); tok != WS { + i++ + } + } +} + +// expect returns an error if the next token is not exp. +func (p *Parser) expect(exp Token) error { + if tok, pos, lit := p.scan(); tok != exp { + return parseErrorf(pos, "expected %s, found %q", exp.String(), lit) + } + return nil +} + +// pos returns the current position. +func (p *Parser) pos() Pos { return p.scanner.pos() } + +// ParseError represents an error that occurred while parsing a PQL query. +type ParseError struct { + Message string + Pos Pos +} + +// Error returns a string representation of e. +func (e *ParseError) Error() string { + return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1) +} + +// parseErrorf returns a formatted parse error. +func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError { + return &ParseError{ + Message: fmt.Sprintf(format, args...), + Pos: pos, + } +} diff --git a/pql/internal/oldpql/parser_test.go b/pql/internal/oldpql/parser_test.go new file mode 100644 index 000000000..7aa8fafe1 --- /dev/null +++ b/pql/internal/oldpql/parser_test.go @@ -0,0 +1,194 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql_test + +import ( + "reflect" + "testing" + + pql "github.com/pilosa/pilosa/pql/internal/oldpql" + _ "github.com/pilosa/pilosa/test" +) + +// Ensure the parser can parse PQL. +func TestParser_Parse(t *testing.T) { + // Parse with no children or arguments. + t.Run("Empty", func(t *testing.T) { + q, err := pql.ParseString(`Bitmap()`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Bitmap", + }, + ) { + t.Fatalf("unexpected call: %s", q.Calls[0]) + } + }) + + // Parse with only children. + t.Run("ChildrenOnly", func(t *testing.T) { + q, err := pql.ParseString(`Union( Bitmap() , Count() )`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Union", + Children: []*pql.Call{ + &pql.Call{Name: "Bitmap"}, + &pql.Call{Name: "Count"}, + }, + }, + ) { + t.Fatalf("unexpected call: %s", q.Calls[0]) + } + }) + + // Parse a single child with a single argument. + t.Run("ChildWithArgument", func(t *testing.T) { + q, err := pql.ParseString(`Count( Bitmap( id=100))`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "Count", + Children: []*pql.Call{ + {Name: "Bitmap", Args: map[string]interface{}{"id": int64(100)}}, + }, + }, + ) { + t.Fatalf("unexpected call: %s", q.Calls[0]) + } + }) + + // Parse with only arguments. + t.Run("ArgumentsOnly", func(t *testing.T) { + q, err := pql.ParseString(`MyCall( key= value, foo="bar", age = 12 , bool0=true, bool1=false, x=null )`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": "value", + "foo": "bar", + "age": int64(12), + "bool0": true, + "bool1": false, + "x": nil, + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse with float arguments. + t.Run("WithFloatArgs", func(t *testing.T) { + q, err := pql.ParseString(`MyCall( key=12.25, foo= 13.167, bar=2., baz=0.9)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": 12.25, + "foo": 13.167, + "bar": 2., + "baz": 0.9, + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse with float arguments. + t.Run("WithNegativeArgs", func(t *testing.T) { + q, err := pql.ParseString(`MyCall( key=-12.25, foo= -13)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": -12.25, + "foo": int64(-13), + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse with both child calls and arguments. + t.Run("ChildrenAndArguments", func(t *testing.T) { + q, err := pql.ParseString(`TopN(Bitmap(id=100, field=other), field=f, n=3)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "TopN", + Children: []*pql.Call{{ + Name: "Bitmap", + Args: map[string]interface{}{"id": int64(100), "field": "other"}, + }}, + Args: map[string]interface{}{"n": int64(3), "field": "f"}, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse a list argument. + t.Run("ListArgument", func(t *testing.T) { + q, err := pql.ParseString(`TopN(field="f", ids=[0,10,30])`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "TopN", + Args: map[string]interface{}{ + "field": "f", + "ids": []interface{}{int64(0), int64(10), int64(30)}, + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + + // Parse with condition arguments. + t.Run("WithCondition", func(t *testing.T) { + q, err := pql.ParseString(`MyCall(key=foo, x == 12.25, y >= 100, z >< [4,8], m != null)`) + if err != nil { + t.Fatal(err) + } else if !reflect.DeepEqual(q.Calls[0], + &pql.Call{ + Name: "MyCall", + Args: map[string]interface{}{ + "key": "foo", + "x": &pql.Condition{Op: pql.EQ, Value: 12.25}, + "y": &pql.Condition{Op: pql.GTE, Value: int64(100)}, + "z": &pql.Condition{Op: pql.BETWEEN, Value: []interface{}{int64(4), int64(8)}}, + "m": &pql.Condition{Op: pql.NEQ, Value: nil}, + }, + }, + ) { + t.Fatalf("unexpected call: %#v", q.Calls[0]) + } + }) + +} diff --git a/pql/internal/oldpql/scanner.go b/pql/internal/oldpql/scanner.go new file mode 100644 index 000000000..dd7f9126d --- /dev/null +++ b/pql/internal/oldpql/scanner.go @@ -0,0 +1,303 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql + +import ( + "bufio" + "bytes" + "io" + "unicode" +) + +// Scanner represents a PQL lexical scanner. +type Scanner struct { + r io.RuneScanner + pos Pos +} + +// NewScanner returns a new instance of Scanner. +func NewScanner(r io.Reader) *Scanner { + return &Scanner{r: bufio.NewReader(r)} +} + +// Scan returns the next token and position from the underlying reader. +func (s *Scanner) Scan() (tok Token, pos Pos, lit string) { + pos = s.pos + + // Read next code point. + ch := s.read() + + // If we see whitespace then consume all contiguous whitespace. + // If we see a letter, or certain acceptable special characters, then consume + // as an ident or reserved word. If we see quotes, then scan as string. + if isWhitespace(ch) { + s.unread() + return s.scanWhitespace() + } else if isIdentFirstChar(ch) { + s.unread() + return s.scanIdent() + } else if isDigit(ch) || ch == '-' { + s.unread() + return s.scanNumber() + } else if ch == '"' || ch == '\'' { + s.unread() + return s.scanString() + } + + // Otherwise parse individual characters. + switch ch { + case eof: + return EOF, pos, "" + case '=': + if next := s.read(); next == '=' { + return EQ, pos, "==" + } + s.unread() + return ASSIGN, pos, string(ch) + case '!': + if next := s.read(); next == '=' { + return NEQ, pos, "!=" + } + s.unread() + return ASSIGN, pos, string(ch) + case '<': + if next := s.read(); next == '=' { + return LTE, pos, "<=" + } + s.unread() + return LT, pos, string(ch) + case '>': + next := s.read() + if next == '=' { + return GTE, pos, ">=" + } else if next == '<' { + return BETWEEN, pos, "><" + } + s.unread() + return GT, pos, string(ch) + case ',': + return COMMA, pos, string(ch) + case '(': + return LPAREN, pos, string(ch) + case ')': + return RPAREN, pos, string(ch) + case '[': + return LBRACK, pos, string(ch) + case ']': + return RBRACK, pos, string(ch) + default: + return ILLEGAL, pos, string(ch) + } +} + +// read returns the next code point from the underlying reader and updates the pos. +func (s *Scanner) read() rune { + // Read next rune from underlying reader. + ch, _, err := s.r.ReadRune() + if err != nil { + return eof + } + + // Update position information. + if ch == '\n' { + s.pos.Line++ + s.pos.Char = 0 + } else { + s.pos.Char++ + } + + return ch +} + +// unread pushes the previously read rune back onto the reader. +func (s *Scanner) unread() { + if s.pos.Char == 0 { + s.pos.Line-- + } else { + s.pos.Char-- + } + + s.r.UnreadRune() +} + +// scanWhitespace consumes the current rune and all contiguous whitespace. +func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) { + pos = s.pos + + var buf bytes.Buffer + for { + ch := s.read() + if ch == eof { + break + } else if !isWhitespace(ch) { + s.unread() + break + } + buf.WriteRune(ch) + } + + return WS, pos, buf.String() +} + +func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) { + pos = s.pos + + var buf bytes.Buffer + for { + ch := s.read() + if ch == eof { + break + } else if !isIdentChar(ch) { + s.unread() + break + } + buf.WriteRune(ch) + } + lit = buf.String() + + // If the literal matches a keyword then return that keyword. + if tok = Lookup(lit); tok != IDENT { + return tok, pos, lit + } + + return IDENT, pos, lit +} + +// scanNumber consumes consecutive digits, optionally starting with a minus sign and up to one '.' character. +func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) { + pos = s.pos + tok = INTEGER + + var buf bytes.Buffer + var seenDot bool + first := true + for { + ch := s.read() + if !isDigit(ch) && !(first && ch == '-') && (seenDot || ch != '.') { + s.unread() + break + } + if ch == '.' { + seenDot = true + tok = FLOAT + } + buf.WriteRune(ch) + first = false + } + return tok, pos, buf.String() +} + +// scanString consumes a single-quoted or double-quoted string. +func (s *Scanner) scanString() (tok Token, pos Pos, lit string) { + pos = s.pos + + // This must be either a single- or double-quote. + ending := s.read() + + var buf bytes.Buffer + for { + ch := s.read() + if ch == ending { + break + } else if ch == '\n' || ch == eof { + return BADSTRING, pos, buf.String() + } else if ch == '\\' { + next := s.read() + if next == 'n' { + buf.WriteRune('\n') + } else if next == '\\' { + buf.WriteRune('\\') + } else if next == '"' { + buf.WriteRune('"') + } else if next == '\'' { + buf.WriteRune('\'') + } else { + return BADSTRING, pos, buf.String() + } + } else { + buf.WriteRune(ch) + } + } + + return STRING, pos, buf.String() +} + +// bufScanner represents a wrapper for scanner to add a buffer. +// It provides a fixed-length circular buffer that can be unread. +type bufScanner struct { + s *Scanner + i int // buffer index + n int // buffer size + buf [8]struct { + tok Token + pos Pos + lit string + } +} + +// newBufScanner returns a new buffered scanner for a reader. +func newBufScanner(r io.Reader) *bufScanner { + return &bufScanner{s: NewScanner(r)} +} + +// Scan reads the next token from the scanner. +func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) { + // If we have unread tokens then read them off the buffer first. + if s.n > 0 { + s.n-- + return s.curr() + } + + // Move buffer position forward and save the token. + s.i = (s.i + 1) % len(s.buf) + buf := &s.buf[s.i] + buf.tok, buf.pos, buf.lit = s.s.Scan() + + return s.curr() +} + +// unscan pushes the previously token back onto the buffer. +func (s *bufScanner) unscan() { s.n++ } + +// curr returns the last read token. +func (s *bufScanner) curr() (tok Token, pos Pos, lit string) { + buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)] + return buf.tok, buf.pos, buf.lit +} + +// pos returns the current position. +func (s *bufScanner) pos() Pos { + _, pos, _ := s.curr() + return pos +} + +// isWhitespace returns true if the rune a Unicode space character. +func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) } + +// isLetter returns true if the rune is a letter. +func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') } + +// isDigit returns true if the rune is a digit. +func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') } + +// isIdentChar returns true if the rune can be used in an unquoted identifier. +func isIdentChar(ch rune) bool { + return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' || ch == '.' +} + +// isIdentFirstChar returns true if the rune can be used as the first char in an identifier. +func isIdentFirstChar(ch rune) bool { return isLetter(ch) } + +const eof = rune(0) diff --git a/pql/internal/oldpql/scanner_test.go b/pql/internal/oldpql/scanner_test.go new file mode 100644 index 000000000..3a1f462e2 --- /dev/null +++ b/pql/internal/oldpql/scanner_test.go @@ -0,0 +1,74 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql_test + +import ( + "strings" + "testing" + + pql "github.com/pilosa/pilosa/pql/internal/oldpql" +) + +func TestScanner_Scan(t *testing.T) { + var tests = []struct { + name string + s string + tok pql.Token + lit string + pos pql.Pos + }{ + // Special tokens (EOF, ILLEGAL, WS) + {name: "EOF", s: ``, tok: pql.EOF}, + {name: "ILLEGAL", s: `#`, tok: pql.ILLEGAL, lit: `#`}, + {name: "WS/SPACE", s: ` `, tok: pql.WS, lit: " "}, + {name: "WS/TAB", s: "\t", tok: pql.WS, lit: "\t"}, + {name: "WS/NEWLINE", s: "\n", tok: pql.WS, lit: "\n"}, + + {name: "ASSIGN", s: `=`, tok: pql.ASSIGN, lit: `=`}, + {name: "EQ", s: `==`, tok: pql.EQ, lit: `==`}, + {name: "NEQ", s: `!=`, tok: pql.NEQ, lit: `!=`}, + {name: "LT", s: `<`, tok: pql.LT, lit: `<`}, + {name: "LTE", s: `<=`, tok: pql.LTE, lit: `<=`}, + {name: "GT", s: `>`, tok: pql.GT, lit: `>`}, + {name: "GTE", s: `>=`, tok: pql.GTE, lit: `>=`}, + {name: "BETWEEN", s: `><`, tok: pql.BETWEEN, lit: `><`}, + {name: "COMMA", s: `,`, tok: pql.COMMA, lit: `,`}, + {name: "LPAREN", s: `(`, tok: pql.LPAREN, lit: `(`}, + {name: "RPAREN", s: `)`, tok: pql.RPAREN, lit: `)`}, + {name: "LBRACK", s: `[`, tok: pql.LBRACK, lit: `[`}, + {name: "RBRACK", s: `]`, tok: pql.RBRACK, lit: `]`}, + + {name: "IDENT", s: `foo`, tok: pql.IDENT, lit: `foo`}, + {name: "INTEGER", s: `100`, tok: pql.INTEGER, lit: `100`}, + {name: "FLOAT", s: `100.3`, tok: pql.FLOAT, lit: `100.3`}, + + {name: "ALL", s: `all`, tok: pql.ALL, lit: `all`}, + {name: "ALL/CASE", s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive + } + + for i, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := pql.NewScanner(strings.NewReader(tt.s)) + tok, pos, lit := s.Scan() + if tt.tok != tok { + t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit) + } else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char { + t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos) + } else if tt.lit != lit { + t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit) + } + }) + } +} diff --git a/pql/internal/oldpql/token.go b/pql/internal/oldpql/token.go new file mode 100644 index 000000000..4da3b8505 --- /dev/null +++ b/pql/internal/oldpql/token.go @@ -0,0 +1,111 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package oldpql + +import "strings" + +// Token is a lexical token of the PQL language. +type Token int + +const ( + // Special tokens + ILLEGAL Token = iota + EOF + WS + + literal_beg + IDENT // main + STRING // "foo" + BADSTRING // bad escape or unclosed string + INTEGER // 12345 + FLOAT // 100.2 + literal_end + + keyword_beg + ALL + keyword_end + + ASSIGN // = + EQ // == + NEQ // != + LT // < + LTE // <= + GT // > + GTE // >= + BETWEEN // >< + COMMA // , + LPAREN // ( + RPAREN // ) + LBRACK // ( + RBRACK // ) +) + +var tokens = [...]string{ + ILLEGAL: "ILLEGAL", + EOF: "EOF", + WS: "WS", + + IDENT: "IDENT", + INTEGER: "INTEGER", + FLOAT: "FLOAT", + + ALL: "ALL", + + ASSIGN: "=", + EQ: "==", + NEQ: "!=", + LT: "<", + LTE: "<=", + GT: ">", + GTE: ">=", + BETWEEN: "><", + COMMA: ",", + LPAREN: "(", + RPAREN: ")", + LBRACK: "(", + RBRACK: ")", +} + +var keywords map[string]Token + +func init() { + keywords = make(map[string]Token) + for tok := keyword_beg + 1; tok < keyword_end; tok++ { + keywords[strings.ToLower(tokens[tok])] = tok + } +} + +// String returns the string representation of the token. +func (tok Token) String() string { + if tok >= 0 && tok < Token(len(tokens)) { + return tokens[tok] + } + return "" +} + +// Lookup returns the token associated with a given string. +func Lookup(ident string) Token { + if tok, ok := keywords[strings.ToLower(ident)]; ok { + return tok + } + return IDENT +} + +// Pos specifies the line and character position of a token. +// The Char and Line are both zero-based indexes. +type Pos struct { + Line int + Char int +}