Merge remote-tracking branch 'origin/master' into metrics

attr.go

@@ -41,7 +41,7 @@ const (
 
 // AttrStore represents a storage layer for attributes.
 type AttrStore struct {
-	mu   sync.Mutex
+	mu   sync.RWMutex
 	path string
 	db   *bolt.DB
 
@@ -92,8 +92,8 @@ func (s *AttrStore) Close() error {
 
 // Attrs returns a set of attributes by ID.
 func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
-	s.mu.Lock()
-	defer s.mu.Unlock()
+	s.mu.RLock()
+	defer s.mu.RUnlock()
 
 	// Check cache for map.
 	if m = s.attrs[id]; m != nil {
@@ -119,6 +119,19 @@ func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) {
 
 // SetAttrs sets attribute values for a given ID.
 func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
+	// Ignore empty maps.
+	if len(m) == 0 {
+		return nil
+	}
+
+	// Check if the attributes already exist under a read-only lock.
+	if attr, err := s.Attrs(id); err != nil {
+		return err
+	} else if attr != nil && mapContains(attr, m) {
+		return nil
+	}
+
+	// Obtain write lock.
 	s.mu.Lock()
 	defer s.mu.Unlock()
 
@@ -493,3 +506,14 @@ func (cur *blockCursor) next() (key, value []byte) {
 
 	return key, value
 }
+
+// mapContains returns true if all keys & values of subset are in m.
+func mapContains(m, subset map[string]interface{}) bool {
+	for k, v := range subset {
+		value, ok := m[k]
+		if !ok || value != v {
+			return false
+		}
+	}
+	return true
+}

attr_test.go

@@ -18,6 +18,8 @@ import (
 	"io/ioutil"
 	"os"
 	"reflect"
+	"runtime"
+	"sync"
 	"testing"
 
 	"github.com/pilosa/pilosa"
@@ -143,6 +145,38 @@ func NewAttrStore() *AttrStore {
 	return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())}
 }
 
+func BenchmarkAttrStore_Duplicate(b *testing.B) {
+	s := MustOpenAttrStore()
+	defer s.Close()
+
+	// Set attributes.
+	const n = 5
+	for i := 0; i < n; i++ {
+		if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
+			b.Fatal(err)
+		}
+	}
+
+	b.ReportAllocs()
+	b.ResetTimer()
+
+	// Update attributes with an existing subset.
+	cpuN := runtime.GOMAXPROCS(0)
+	var wg sync.WaitGroup
+	for i := 0; i < cpuN; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for j := 0; j < b.N/cpuN; j++ {
+				if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
+					b.Fatal(err)
+				}
+			}
+		}()
+	}
+	wg.Wait()
+}
+
 // MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
 func MustOpenAttrStore() *AttrStore {
 	s := NewAttrStore()

cmd/root.go

@@ -95,6 +95,11 @@ func setAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error
 	v.AutomaticEnv()
 
 	c := v.GetString("config")
+	var flagErr error
+	validTags := make(map[string]bool)
+	flags.VisitAll(func(f *pflag.Flag) {
+		validTags[f.Name] = true
+	})
 
 	// add config file to viper
 	if c != "" {
@@ -104,10 +109,16 @@ func setAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error
 		if err != nil {
 			return fmt.Errorf("error reading configuration file '%s': %v", c, err)
 		}
+
+		for _, key := range v.AllKeys() {
+			if _, ok := validTags[key]; !ok {
+				return fmt.Errorf("invalid option in configuration file: %v", key)
+			}
+		}
+
 	}
 
 	// set all values from viper
-	var flagErr error
 	flags.VisitAll(func(f *pflag.Flag) {
 		if flagErr != nil {
 			return

cmd/root_test.go

@@ -171,3 +171,27 @@ func TestRootCommand(t *testing.T) {
 		t.Fatalf("Expected standard usage message from RootCommand, but err: '%v', output: '%s'", err, outStr)
 	}
 }
+
+func TestRootCommand_Config(t *testing.T) {
+	file, err := ioutil.TempFile("", "test.conf")
+	if err != nil {
+		panic(err)
+	}
+	config := `data-dir = "/tmp/pil5_0"
+bind = "127.0.0.1:10101"
+
+[cluster]
+  poll-interval = "2m0s"
+  replicas = 2
+  partitions = 128
+  hosts = [
+    "127.0.0.1:10101",
+    "127.0.0.1:10111",
+  ]`
+	file.Write([]byte(config))
+	file.Close()
+	_, err = ExecNewRootCommand(t, "server", "--config", file.Name())
+	if err.Error() != "invalid option in configuration file: cluster.partitions" {
+		t.Fatalf("Expected invalid option in configuration file, but err: '%v'", err)
+	}
+}

ctl/export.go

@@ -31,7 +31,7 @@ type ExportCommand struct {
 	// Name of the index & frame to export from.
 	Index string
 	Frame string
-	View string
+	View  string
 	// Filename to export to.
 	Path string
 

fragment.go

@@ -86,6 +86,9 @@ type Fragment struct {
 	cache     Cache
 	CacheSize uint32
 
+	// Stats reporting.
+	maxRowID uint64
+
 	// Cache containing full rows (not just counts).
 	rowCache BitmapCache
 
@@ -166,6 +169,11 @@ func (f *Fragment) Open() error {
 		// Clear checksums.
 		f.checksums = make(map[int][]byte)
 
+		// Read last bit to determine max row.
+		pos := f.storage.Max()
+		f.maxRowID = pos / SliceWidth
+		f.stats.Gauge("rows", float64(f.maxRowID))
+
 		return nil
 	}(); err != nil {
 		f.close()
@@ -409,6 +417,12 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
 
 	f.stats.Count("setBit", 1, 0.001)
 
+	// Update row count if they have increased.
+	if rowID > f.maxRowID {
+		f.maxRowID = rowID
+		f.stats.Gauge("rows", float64(f.maxRowID))
+	}
+
 	return changed, nil
 }
 

handler.go

@@ -42,6 +42,7 @@ import (
 
 	_ "github.com/pilosa/pilosa/statik"
 	"github.com/rakyll/statik/fs"
+	"unicode"
 )
 
 // Handler represents an HTTP handler.
@@ -885,6 +886,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, erro
 // readURLQueryRequest parses query parameters from URL parameters from r.
 func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
 	q := r.URL.Query()
+	validQuery := validOptions(QueryRequest{})
+	for key, _ := range q {
+		if _, ok := validQuery[key]; !ok {
+			return nil, errors.New("invalid query params")
+		}
+	}
 
 	// Parse query string.
 	buf, err := ioutil.ReadAll(r.Body)
@@ -917,6 +924,21 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
 	}, nil
 }
 
+// validOptions return all attributes of an interface with lower first character.
+func validOptions(v interface{}) map[string]bool {
+	validQuery := make(map[string]bool)
+	argsType := reflect.ValueOf(v).Type()
+
+	for i := 0; i < argsType.NumField(); i++ {
+		fieldName := argsType.Field(i).Name
+		chars := []rune(fieldName)
+		chars[0] = unicode.ToLower(chars[0])
+		fieldName = string(chars)
+		validQuery[fieldName] = true
+	}
+	return validQuery
+}
+
 // writeQueryResponse writes the response from the executor to w.
 func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error {
 	if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") {

handler_test.go

@@ -277,6 +277,16 @@ func TestHandler_Query_Args_Err(t *testing.T) {
 		t.Fatalf("unexpected body: %q", body)
 	}
 }
+func TestHandler_Query_Params_Err(t *testing.T) {
+	w := httptest.NewRecorder()
+	NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)")))
+	if w.Code != http.StatusBadRequest {
+		t.Fatalf("unexpected status code: %d", w.Code)
+	} else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" {
+		t.Fatalf("unexpected body: %q", body)
+	}
+
+}
 
 // Ensure the handler can execute a query with a uint64 response as JSON.
 func TestHandler_Query_Uint64_JSON(t *testing.T) {

pilosa.go

@@ -49,10 +49,10 @@ var (
 )
 
 // Regular expression to validate index and frame names.
-var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,64}$`)
+var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`)
 
 // Regular expression to validate row and column labels.
-var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,64}$`)
+var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`)
 
 // ColumnAttrSet represents a set of attributes for a vertical column in an index.
 // Can have a set of attributes attached to it.

pilosa_test.go

@@ -0,0 +1,55 @@
+package pilosa_test
+
+import (
+	"testing"
+
+	"github.com/pilosa/pilosa"
+)
+
+func TestValidateName(t *testing.T) {
+	names := []string{
+		"a", "ab", "ab1", "b-c", "d_e",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+	}
+	for _, name := range names {
+		if pilosa.ValidateName(name) != nil {
+			t.Fatalf("Should be valid index name: %s", name)
+		}
+	}
+}
+
+func TestValidateNameInvalid(t *testing.T) {
+	names := []string{
+		"", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+	}
+	for _, name := range names {
+		if pilosa.ValidateName(name) == nil {
+			t.Fatalf("Should be invalid index name: %s", name)
+		}
+	}
+}
+
+func TestValidateLabel(t *testing.T) {
+	labels := []string{
+		"a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+	}
+	for _, label := range labels {
+		if pilosa.ValidateLabel(label) != nil {
+			t.Fatalf("Should be valid label: %s", label)
+		}
+	}
+}
+
+func TestValidateLabelInvalid(t *testing.T) {
+	labels := []string{
+		"", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce",
+		"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1",
+	}
+	for _, label := range labels {
+		if pilosa.ValidateLabel(label) == nil {
+			t.Fatalf("Should be invalid label: %s", label)
+		}
+	}
+}

pql/ast.go

@@ -72,7 +72,7 @@ func (c *Call) UintArg(key string) (uint64, bool, error) {
 	case uint64:
 		return tval, true, nil
 	default:
-		return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Calll.UintArg", tval, tval)
+		return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.UintArg", tval, tval)
 	}
 }
 

server.go

@@ -105,7 +105,7 @@ func (s *Server) Open() error {
 	// Open HTTP listener to determine port (if specified as :0).
 	ln, err := net.Listen("tcp", ":"+port)
 	if err != nil {
-		return err
+		return fmt.Errorf("net.Listen: %v", err)
 	}
 	s.ln = ln
 
@@ -125,16 +125,16 @@ func (s *Server) Open() error {
 
 	// Open holder.
 	if err := s.Holder.Open(); err != nil {
-		return err
+		return fmt.Errorf("opening Holder: %v", err)
 	}
 
 	if err := s.BroadcastReceiver.Start(s); err != nil {
-		return err
+		return fmt.Errorf("starting BroadcastReceiver: %v", err)
 	}
 
 	// Open NodeSet communication
 	if err := s.Cluster.NodeSet.Open(); err != nil {
-		return err
+		return fmt.Errorf("opening NodeSet: %v", err)
 	}
 
 	// Create executor for executing queries.

webui/assets/main.js

@@ -1,8 +1,9 @@
 class REPL {
-    constructor(input, output, button) {
+    constructor(input, output, button, completer) {
         this.input = input
         this.output = output
         this.button = button
+        this.completer = completer
         this.history = []
         this.history_index = 0
         this.history_buffer = ''
@@ -16,21 +17,6 @@ class REPL {
           UP_ARROW: 38,
           DOWN_ARROW: 40
         }
-        var keywords = {
-          // keyword: length of substring that comes after cursor
-          "SetBit()": 1,
-          "ClearBit()": 1,
-          "SetRowAttrs()": 1,
-          "SetColumnAttrs()": 1,
-          "Bitmap()": 1,
-          "Union()": 1,
-          "Intersect()": 1,
-          "Difference()": 1,
-          "Count()": 1,
-          "Range()": 1,
-          "TopN()": 1,
-          "frame=": 0,
-        }
 
         this.input.addEventListener("keydown", function(e) {
           if (e.keyCode == keys.UP_ARROW) {
@@ -70,35 +56,7 @@ class REPL {
           }
           if (e.keyCode == keys.TAB) {
             e.preventDefault()
-
-            // extract word fragment ending at cursor. a word fragment:
-            // - starts with last nonalpha character before cursor (or beginning of string)
-            // - ends at cursor
-            var word_start = repl.input.selectionEnd-1
-            while(word_start>0) {
-              var c = repl.input.value.charCodeAt(word_start)
-              if(!((c>64 && c<91) || (c>96 && c<123))) {
-                word_start++
-                break
-              }
-              word_start--
-            }
-            var input_word = repl.input.value.substring(word_start, repl.input.selectionEnd)
-
-            // check for keyword match and insert
-            // this just stops at the first match
-            for(var keyword in keywords) {
-              if(keyword.startsWith(input_word)){
-                var cursor_pos = repl.input.selectionEnd
-                var completion = keyword.substring(input_word.length)
-                var before = repl.input.value.substring(0, cursor_pos)
-                var after = repl.input.value.substring(cursor_pos)
-                repl.input.value = before + completion + after
-                var new_pos = cursor_pos + completion.length - keywords[keyword]
-                repl.input.setSelectionRange(new_pos, new_pos)
-                break
-              }
-            }
+            repl.completer.complete()
           }
         })
         repl.button.onclick = function() {
@@ -444,11 +402,94 @@ Date.prototype.timeNow = function () {
 
 populate_version()
 
+
+class Autocompleter {
+  constructor(input, output) {
+    this.input = input
+    this.output = output
+    this.keyword_map = this.static_keywords
+    this.init_dynamic_keywords()
+  }
+
+  get static_keywords() {
+    return {
+      // keyword: length of substring that comes after cursor
+      "SetBit()": 1,
+      "ClearBit()": 1,
+      "SetRowAttrs()": 1,
+      "SetColumnAttrs()": 1,
+      "Bitmap()": 1,
+      "Union()": 1,
+      "Intersect()": 1,
+      "Difference()": 1,
+      "Count()": 1,
+      "Range()": 1,
+      "TopN()": 1,
+      "frame=": 0,
+    }
+  }
+
+  complete() {
+    var completer = this
+    // extract word fragment ending at cursor. a word fragment:
+    // - starts with last nonalpha character before cursor (or beginning of string)
+    // - ends at cursor
+    var word_start = completer.input.selectionEnd-1
+    while(word_start>0) {
+      var c = completer.input.value.charCodeAt(word_start)
+      if(!((c>64 && c<91) || (c>96 && c<123))) {
+        word_start++
+        break
+      }
+      word_start--
+    }
+    var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd)
+
+    // check for keyword match and insert if exactly one match
+    var matches = []
+    for(var keyword in this.keyword_map) {
+      if(keyword.startsWith(input_word)){
+        matches.push(keyword)
+      }
+    }
+    if(matches.length > 1) {
+      // completer.output.innerHTML = whatever
+    }
+
+    if(matches.length == 1) {
+      // completer.output.innerHTML = ""
+      var cursor_pos = completer.input.selectionEnd
+      var completion = matches[0].substring(input_word.length)
+      var before = completer.input.value.substring(0, cursor_pos)
+      var after = completer.input.value.substring(cursor_pos)
+      completer.input.value = before + completion + after
+      var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]]
+      completer.input.setSelectionRange(new_pos, new_pos)
+    }
+  }
+
+  init_dynamic_keywords() {
+    // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list
+  }
+
+  add_keyword() {
+    // call when index or frame created in webui
+  }
+
+  remove_keyword() {
+    // call when index or frame deleted in webui
+    // issue: if e.g. multiple indexes have same frame, removing one removes all.
+    // solution: maintain count. requires more elaborate representation of keywords.
+  }
+}
+
 var input = document.getElementById('query')
 var output = document.getElementById('outputs')
 var button = document.getElementById('query-btn')
+var autocomplete_output = document.getElementById('autocomplete-container')
 
-repl = new REPL(input, output, button)
+autocompleter = new Autocompleter(input, autocomplete_output)
+repl = new REPL(input, output, button, autocompleter)
 repl.populate_index_dropdown()
 repl.bind_events()