diff --git a/attr.go b/attr.go index 2e45aadf8..6f12ffdc0 100644 --- a/attr.go +++ b/attr.go @@ -41,7 +41,7 @@ const ( // AttrStore represents a storage layer for attributes. type AttrStore struct { - mu sync.Mutex + mu sync.RWMutex path string db *bolt.DB @@ -92,8 +92,8 @@ func (s *AttrStore) Close() error { // Attrs returns a set of attributes by ID. func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { - s.mu.Lock() - defer s.mu.Unlock() + s.mu.RLock() + defer s.mu.RUnlock() // Check cache for map. if m = s.attrs[id]; m != nil { @@ -119,6 +119,19 @@ func (s *AttrStore) Attrs(id uint64) (m map[string]interface{}, err error) { // SetAttrs sets attribute values for a given ID. func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { + // Ignore empty maps. + if len(m) == 0 { + return nil + } + + // Check if the attributes already exist under a read-only lock. + if attr, err := s.Attrs(id); err != nil { + return err + } else if attr != nil && mapContains(attr, m) { + return nil + } + + // Obtain write lock. s.mu.Lock() defer s.mu.Unlock() @@ -493,3 +506,14 @@ func (cur *blockCursor) next() (key, value []byte) { return key, value } + +// mapContains returns true if all keys & values of subset are in m. +func mapContains(m, subset map[string]interface{}) bool { + for k, v := range subset { + value, ok := m[k] + if !ok || value != v { + return false + } + } + return true +} diff --git a/attr_test.go b/attr_test.go index dda280214..d9884cedc 100644 --- a/attr_test.go +++ b/attr_test.go @@ -18,6 +18,8 @@ import ( "io/ioutil" "os" "reflect" + "runtime" + "sync" "testing" "github.com/pilosa/pilosa" @@ -143,6 +145,38 @@ func NewAttrStore() *AttrStore { return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())} } +func BenchmarkAttrStore_Duplicate(b *testing.B) { + s := MustOpenAttrStore() + defer s.Close() + + // Set attributes. + const n = 5 + for i := 0; i < n; i++ { + if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil { + b.Fatal(err) + } + } + + b.ReportAllocs() + b.ResetTimer() + + // Update attributes with an existing subset. + cpuN := runtime.GOMAXPROCS(0) + var wg sync.WaitGroup + for i := 0; i < cpuN; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for j := 0; j < b.N/cpuN; j++ { + if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil { + b.Fatal(err) + } + } + }() + } + wg.Wait() +} + // MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error. func MustOpenAttrStore() *AttrStore { s := NewAttrStore() diff --git a/cmd/root.go b/cmd/root.go index 228a623e4..d805d519f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -95,6 +95,11 @@ func setAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error v.AutomaticEnv() c := v.GetString("config") + var flagErr error + validTags := make(map[string]bool) + flags.VisitAll(func(f *pflag.Flag) { + validTags[f.Name] = true + }) // add config file to viper if c != "" { @@ -104,10 +109,16 @@ func setAllConfig(v *viper.Viper, flags *pflag.FlagSet, envPrefix string) error if err != nil { return fmt.Errorf("error reading configuration file '%s': %v", c, err) } + + for _, key := range v.AllKeys() { + if _, ok := validTags[key]; !ok { + return fmt.Errorf("invalid option in configuration file: %v", key) + } + } + } // set all values from viper - var flagErr error flags.VisitAll(func(f *pflag.Flag) { if flagErr != nil { return diff --git a/cmd/root_test.go b/cmd/root_test.go index 16bc82a8a..e1aaaf31d 100644 --- a/cmd/root_test.go +++ b/cmd/root_test.go @@ -171,3 +171,27 @@ func TestRootCommand(t *testing.T) { t.Fatalf("Expected standard usage message from RootCommand, but err: '%v', output: '%s'", err, outStr) } } + +func TestRootCommand_Config(t *testing.T) { + file, err := ioutil.TempFile("", "test.conf") + if err != nil { + panic(err) + } + config := `data-dir = "/tmp/pil5_0" +bind = "127.0.0.1:10101" + +[cluster] + poll-interval = "2m0s" + replicas = 2 + partitions = 128 + hosts = [ + "127.0.0.1:10101", + "127.0.0.1:10111", + ]` + file.Write([]byte(config)) + file.Close() + _, err = ExecNewRootCommand(t, "server", "--config", file.Name()) + if err.Error() != "invalid option in configuration file: cluster.partitions" { + t.Fatalf("Expected invalid option in configuration file, but err: '%v'", err) + } +} diff --git a/ctl/export.go b/ctl/export.go index fa6277f16..295e10039 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -31,7 +31,7 @@ type ExportCommand struct { // Name of the index & frame to export from. Index string Frame string - View string + View string // Filename to export to. Path string diff --git a/fragment.go b/fragment.go index 0b209f218..2d869c754 100644 --- a/fragment.go +++ b/fragment.go @@ -86,6 +86,9 @@ type Fragment struct { cache Cache CacheSize uint32 + // Stats reporting. + maxRowID uint64 + // Cache containing full rows (not just counts). rowCache BitmapCache @@ -166,6 +169,11 @@ func (f *Fragment) Open() error { // Clear checksums. f.checksums = make(map[int][]byte) + // Read last bit to determine max row. + pos := f.storage.Max() + f.maxRowID = pos / SliceWidth + f.stats.Gauge("rows", float64(f.maxRowID)) + return nil }(); err != nil { f.close() @@ -409,6 +417,12 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { f.stats.Count("setBit", 1, 0.001) + // Update row count if they have increased. + if rowID > f.maxRowID { + f.maxRowID = rowID + f.stats.Gauge("rows", float64(f.maxRowID)) + } + return changed, nil } diff --git a/handler.go b/handler.go index a36e1e907..6cbf693ef 100644 --- a/handler.go +++ b/handler.go @@ -42,6 +42,7 @@ import ( _ "github.com/pilosa/pilosa/statik" "github.com/rakyll/statik/fs" + "unicode" ) // Handler represents an HTTP handler. @@ -885,6 +886,12 @@ func (h *Handler) readProtobufQueryRequest(r *http.Request) (*QueryRequest, erro // readURLQueryRequest parses query parameters from URL parameters from r. func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { q := r.URL.Query() + validQuery := validOptions(QueryRequest{}) + for key, _ := range q { + if _, ok := validQuery[key]; !ok { + return nil, errors.New("invalid query params") + } + } // Parse query string. buf, err := ioutil.ReadAll(r.Body) @@ -917,6 +924,21 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { }, nil } +// validOptions return all attributes of an interface with lower first character. +func validOptions(v interface{}) map[string]bool { + validQuery := make(map[string]bool) + argsType := reflect.ValueOf(v).Type() + + for i := 0; i < argsType.NumField(); i++ { + fieldName := argsType.Field(i).Name + chars := []rune(fieldName) + chars[0] = unicode.ToLower(chars[0]) + fieldName = string(chars) + validQuery[fieldName] = true + } + return validQuery +} + // writeQueryResponse writes the response from the executor to w. func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, resp *QueryResponse) error { if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { diff --git a/handler_test.go b/handler_test.go index b46821d93..ab65894fb 100644 --- a/handler_test.go +++ b/handler_test.go @@ -277,6 +277,16 @@ func TestHandler_Query_Args_Err(t *testing.T) { t.Fatalf("unexpected body: %q", body) } } +func TestHandler_Query_Params_Err(t *testing.T) { + w := httptest.NewRecorder() + NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)"))) + if w.Code != http.StatusBadRequest { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" { + t.Fatalf("unexpected body: %q", body) + } + +} // Ensure the handler can execute a query with a uint64 response as JSON. func TestHandler_Query_Uint64_JSON(t *testing.T) { diff --git a/pilosa.go b/pilosa.go index a192364ed..13ade19ea 100644 --- a/pilosa.go +++ b/pilosa.go @@ -49,10 +49,10 @@ var ( ) // Regular expression to validate index and frame names. -var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,64}$`) +var nameRegexp = regexp.MustCompile(`^[a-z][a-z0-9_-]{0,63}$`) // Regular expression to validate row and column labels. -var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,64}$`) +var labelRegexp = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9_-]{0,63}$`) // ColumnAttrSet represents a set of attributes for a vertical column in an index. // Can have a set of attributes attached to it. diff --git a/pilosa_test.go b/pilosa_test.go new file mode 100644 index 000000000..ad5f9b762 --- /dev/null +++ b/pilosa_test.go @@ -0,0 +1,55 @@ +package pilosa_test + +import ( + "testing" + + "github.com/pilosa/pilosa" +) + +func TestValidateName(t *testing.T) { + names := []string{ + "a", "ab", "ab1", "b-c", "d_e", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, name := range names { + if pilosa.ValidateName(name) != nil { + t.Fatalf("Should be valid index name: %s", name) + } + } +} + +func TestValidateNameInvalid(t *testing.T) { + names := []string{ + "", "'", "^", "/", "\\", "A", "*", "a:b", "valid?no", "yüce", "1", "_", "-", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", + } + for _, name := range names { + if pilosa.ValidateName(name) == nil { + t.Fatalf("Should be invalid index name: %s", name) + } + } +} + +func TestValidateLabel(t *testing.T) { + labels := []string{ + "a", "ab", "ab1", "d_e", "A", "Bc", "B1", "aB", "b-c", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", + } + for _, label := range labels { + if pilosa.ValidateLabel(label) != nil { + t.Fatalf("Should be valid label: %s", label) + } + } +} + +func TestValidateLabelInvalid(t *testing.T) { + labels := []string{ + "", "1", "_", "-", "'", "^", "/", "\\", "*", "a:b", "valid?no", "yüce", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa1", + } + for _, label := range labels { + if pilosa.ValidateLabel(label) == nil { + t.Fatalf("Should be invalid label: %s", label) + } + } +} diff --git a/pql/ast.go b/pql/ast.go index cf404ad12..13a989b03 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -72,7 +72,7 @@ func (c *Call) UintArg(key string) (uint64, bool, error) { case uint64: return tval, true, nil default: - return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Calll.UintArg", tval, tval) + return 0, true, fmt.Errorf("could not convert %v of type %T to uint64 in Call.UintArg", tval, tval) } } diff --git a/server.go b/server.go index 78dc7644a..84314cebe 100644 --- a/server.go +++ b/server.go @@ -105,7 +105,7 @@ func (s *Server) Open() error { // Open HTTP listener to determine port (if specified as :0). ln, err := net.Listen("tcp", ":"+port) if err != nil { - return err + return fmt.Errorf("net.Listen: %v", err) } s.ln = ln @@ -125,16 +125,16 @@ func (s *Server) Open() error { // Open holder. if err := s.Holder.Open(); err != nil { - return err + return fmt.Errorf("opening Holder: %v", err) } if err := s.BroadcastReceiver.Start(s); err != nil { - return err + return fmt.Errorf("starting BroadcastReceiver: %v", err) } // Open NodeSet communication if err := s.Cluster.NodeSet.Open(); err != nil { - return err + return fmt.Errorf("opening NodeSet: %v", err) } // Create executor for executing queries. diff --git a/webui/assets/main.js b/webui/assets/main.js index 887181338..db54a3b1e 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -1,8 +1,9 @@ class REPL { - constructor(input, output, button) { + constructor(input, output, button, completer) { this.input = input this.output = output this.button = button + this.completer = completer this.history = [] this.history_index = 0 this.history_buffer = '' @@ -16,21 +17,6 @@ class REPL { UP_ARROW: 38, DOWN_ARROW: 40 } - var keywords = { - // keyword: length of substring that comes after cursor - "SetBit()": 1, - "ClearBit()": 1, - "SetRowAttrs()": 1, - "SetColumnAttrs()": 1, - "Bitmap()": 1, - "Union()": 1, - "Intersect()": 1, - "Difference()": 1, - "Count()": 1, - "Range()": 1, - "TopN()": 1, - "frame=": 0, - } this.input.addEventListener("keydown", function(e) { if (e.keyCode == keys.UP_ARROW) { @@ -70,35 +56,7 @@ class REPL { } if (e.keyCode == keys.TAB) { e.preventDefault() - - // extract word fragment ending at cursor. a word fragment: - // - starts with last nonalpha character before cursor (or beginning of string) - // - ends at cursor - var word_start = repl.input.selectionEnd-1 - while(word_start>0) { - var c = repl.input.value.charCodeAt(word_start) - if(!((c>64 && c<91) || (c>96 && c<123))) { - word_start++ - break - } - word_start-- - } - var input_word = repl.input.value.substring(word_start, repl.input.selectionEnd) - - // check for keyword match and insert - // this just stops at the first match - for(var keyword in keywords) { - if(keyword.startsWith(input_word)){ - var cursor_pos = repl.input.selectionEnd - var completion = keyword.substring(input_word.length) - var before = repl.input.value.substring(0, cursor_pos) - var after = repl.input.value.substring(cursor_pos) - repl.input.value = before + completion + after - var new_pos = cursor_pos + completion.length - keywords[keyword] - repl.input.setSelectionRange(new_pos, new_pos) - break - } - } + repl.completer.complete() } }) repl.button.onclick = function() { @@ -444,11 +402,94 @@ Date.prototype.timeNow = function () { populate_version() + +class Autocompleter { + constructor(input, output) { + this.input = input + this.output = output + this.keyword_map = this.static_keywords + this.init_dynamic_keywords() + } + + get static_keywords() { + return { + // keyword: length of substring that comes after cursor + "SetBit()": 1, + "ClearBit()": 1, + "SetRowAttrs()": 1, + "SetColumnAttrs()": 1, + "Bitmap()": 1, + "Union()": 1, + "Intersect()": 1, + "Difference()": 1, + "Count()": 1, + "Range()": 1, + "TopN()": 1, + "frame=": 0, + } + } + + complete() { + var completer = this + // extract word fragment ending at cursor. a word fragment: + // - starts with last nonalpha character before cursor (or beginning of string) + // - ends at cursor + var word_start = completer.input.selectionEnd-1 + while(word_start>0) { + var c = completer.input.value.charCodeAt(word_start) + if(!((c>64 && c<91) || (c>96 && c<123))) { + word_start++ + break + } + word_start-- + } + var input_word = completer.input.value.substring(word_start, completer.input.selectionEnd) + + // check for keyword match and insert if exactly one match + var matches = [] + for(var keyword in this.keyword_map) { + if(keyword.startsWith(input_word)){ + matches.push(keyword) + } + } + if(matches.length > 1) { + // completer.output.innerHTML = whatever + } + + if(matches.length == 1) { + // completer.output.innerHTML = "" + var cursor_pos = completer.input.selectionEnd + var completion = matches[0].substring(input_word.length) + var before = completer.input.value.substring(0, cursor_pos) + var after = completer.input.value.substring(cursor_pos) + completer.input.value = before + completion + after + var new_pos = cursor_pos + completion.length - this.keyword_map[matches[0]] + completer.input.setSelectionRange(new_pos, new_pos) + } + } + + init_dynamic_keywords() { + // hit /schema, parse indexes, frames, rowlabels, columnlabels, add to list + } + + add_keyword() { + // call when index or frame created in webui + } + + remove_keyword() { + // call when index or frame deleted in webui + // issue: if e.g. multiple indexes have same frame, removing one removes all. + // solution: maintain count. requires more elaborate representation of keywords. + } +} + var input = document.getElementById('query') var output = document.getElementById('outputs') var button = document.getElementById('query-btn') +var autocomplete_output = document.getElementById('autocomplete-container') -repl = new REPL(input, output, button) +autocompleter = new Autocompleter(input, autocomplete_output) +repl = new REPL(input, output, button, autocompleter) repl.populate_index_dropdown() repl.bind_events()