diff --git a/.gitignore b/.gitignore index 25442a3da..cd99c1b06 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ default.etcd/ *.test +vendor diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..4d5c27fd2 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,14 @@ +language: go +go: + - 1.7.x + - master +addons: + apt: + sources: + - sourceline: 'ppa:masterminds/glide' + packages: + - glide +script: make test +notifications: + slack: + secure: k0m8IqQIjb8L5b1plR9+w4hGo5iWlGtrR9nxvPsHL7gShwez74dpHCmFUCjR/uJoZa5Xq6849IWmiBB71ZHE/wNVN60HpMeui9VKkYvUDVpgBuqoIo969ySPDyQo4jNsk0wRRv/dG5/wXo7ArY8qOp5UDjdVuEf4nEbDlBtKjo0A2W8qXBTdZ5EJ3VeP5nrtrT7knuWxXgeUyVmLkpnUaLgkY7icvH6evqkftfcb56n3l5ulKtlQkI/ij2XCUK+5xtNB9m0e1QbiqIFS7sjwlSly/GCstPVnTEp4oHNHE7LtS6TJ8lgRV90Qy6LPtROjbSdfVjhJ5cYKus8DdddSfcsm2qgHCHQThyYtVAfOXaFPYPb50WXEu3HOckHk1tYChfkdxwEGdn/cN8/xxa7Khd7T/+SaS1B4CPxOFp0ie+rD4lF6IhApmpr/UwtiW6srHcPB04lF5mioYWmyS2eyGKTzSZR82nifg5KYUgBL1FjywQDYb8t9QaAn/6svu08k4H29TBnYpOQWREb38pYmEnMtRqiRmbfQL6IyV8405TJkl/PZmIElX6IQGjf53M1KoFdCUuTL1we6rh8gen5zRVUwmM6BJxkVt4NJ2wGkrXg/HD3feYCMrv+DY3lfDN7tADx90lPb+ukmanB8oSIQHLuLCJ2XcFixEC3QSvVlnH0= diff --git a/Makefile b/Makefile index 61abf2026..09b5616be 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,47 @@ -.PHONY: vendor +.PHONY: glide vendor-update docker pilosa pilosactl crossbuild install -default: +GLIDE := $(shell command -v glide 2>/dev/null) +VERSION := $(shell git describe --tags) +IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH) +CLONE_URL=github.com/pilosa/pilosa +BUILD_TIME=`date -u +%FT%T%z` +LDFLAGS=-ldflags "-X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME)" -vendor: - godep save ./... - cp -r $(GOPATH)/src/github.com/gogo/protobuf/proto/testdata vendor/github.com/gogo/protobuf/proto/testdata +default: test pilosa pilosactl -docker: +$(GOPATH)/bin: + mkdir $(GOPATH)/bin + +glide: $(GOPATH)/bin +ifndef GLIDE + curl https://glide.sh/get | sh +endif + +vendor: glide glide.yaml + glide install + +glide.lock: glide glide.yaml + glide update + +vendor-update: glide.lock + +test: vendor + go test $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor) + +pilosa: vendor + go build $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + +pilosactl: vendor + go build $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosactl + +crossbuild: vendor + mkdir -p build/pilosa-$(IDENTIFIER) + make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa" + make pilosactl FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosactl" + +install: vendor + go install $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa + go install $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosactl + +docker: vendor docker build -t pilosa:latest . diff --git a/README.md b/README.md index 2079010ec..bd6aacbe2 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,7 @@ Pilosa is a bitmap index database. +[![Build Status](https://travis-ci.com/pilosa/pilosa.svg?token=Peb4jvQ3kLbjUEhpU5aR&branch=master)](https://travis-ci.com/pilosa/pilosa) ## Getting Started diff --git a/client.go b/client.go index bc6338053..bfe4dc57e 100644 --- a/client.go +++ b/client.go @@ -108,6 +108,51 @@ func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) { return rsp.DBs, nil } +// CreateDB creates a new database on the server. +func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error { + // Encode query request. + buf, err := json.Marshal(&postDBRequest{ + DB: db, + Options: opt, + }) + if err != nil { + return err + } + + // Create URL & HTTP request. + u := url.URL{Scheme: "http", Host: c.host, Path: "/db"} + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return err + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return err + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return ErrDatabaseExists + default: + return errors.New(string(body)) + } +} + // FragmentNodes returns a list of nodes that own a slice. func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([]*Node, error) { // Execute request against the host. @@ -606,6 +651,56 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame str return nil } +// CreateFrame creates a new frame on the server. +func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOptions) error { + if db == "" { + return ErrDatabaseRequired + } + + // Encode query request. + buf, err := json.Marshal(&postFrameRequest{ + DB: db, + Frame: frame, + Options: opt, + }) + if err != nil { + return err + } + + // Create URL & HTTP request. + u := url.URL{Scheme: "http", Host: c.host, Path: "/frame"} + req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) + if err != nil { + return err + } + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "application/json") + + // Execute request against the host. + resp, err := c.HTTPClient.Do(req.WithContext(ctx)) + if err != nil { + return err + } + defer resp.Body.Close() + + // Read body. + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + + // Handle response based on status code. + switch resp.StatusCode { + case http.StatusOK: + return nil // ok + case http.StatusConflict: + return ErrFrameExists + default: + return errors.New(string(body)) + } +} + // RestoreFrame restores an entire frame from a host in another cluster. func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error { u := url.URL{ @@ -807,6 +902,8 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At // Return error if status is not OK. switch resp.StatusCode { case http.StatusOK: // ok + case http.StatusNotFound: + return nil, ErrFrameNotFound default: return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode) } diff --git a/client_test.go b/client_test.go index f37fea6ee..aa8b3b126 100644 --- a/client_test.go +++ b/client_test.go @@ -71,6 +71,9 @@ func TestClient_BackupRestore(t *testing.T) { } // Restore to a different frame. + if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } if err := c.RestoreFrom(context.Background(), &buf, "x", "y"); err != nil { t.Fatal(err) } diff --git a/cmd/pilosa/main.go b/cmd/pilosa/main.go index bcd3ee453..4f7474996 100644 --- a/cmd/pilosa/main.go +++ b/cmd/pilosa/main.go @@ -17,13 +17,19 @@ import ( "github.com/pilosa/pilosa" ) -// Version holds the version information passed in at compile time. -var Version string +// Version and BuildTime hold the version/build time information passed in at compile time. +var ( + Version string + BuildTime string +) func init() { if Version == "" { Version = "v0.0.0" } + if BuildTime == "" { + BuildTime = "not recorded" + } rand.Seed(time.Now().UTC().UnixNano()) } @@ -36,7 +42,7 @@ const ( func main() { m := NewMain() m.Server.Handler.Version = Version - fmt.Fprintf(m.Stderr, "Pilosa %s\n", Version) + fmt.Fprintf(m.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime) // Parse command line arguments. if err := m.ParseFlags(os.Args[1:]); err != nil { diff --git a/cmd/pilosa/main_test.go b/cmd/pilosa/main_test.go index 1267d44a5..184f5c334 100644 --- a/cmd/pilosa/main_test.go +++ b/cmd/pilosa/main_test.go @@ -31,8 +31,20 @@ func TestMain_Set_Quick(t *testing.T) { m := MustRunMain() defer m.Close() + // Create client. + client, err := pilosa.NewClient(m.Server.Host) + if err != nil { + t.Fatal(err) + } + // Execute SetBit() commands. for _, cmd := range cmds { + if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + t.Fatal(err) + } + if err := client.CreateFrame(context.Background(), "d", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { + t.Fatal(err) + } if _, err := m.Query("db=d", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil { t.Fatal(err) } @@ -95,6 +107,16 @@ func TestMain_SetBitmapAttrs(t *testing.T) { m := MustRunMain() defer m.Close() + // Create frames. + client := m.Client() + if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + t.Fatal(err) + } else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if err := client.CreateFrame(context.Background(), "d", "z", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + // Set bits on different bitmaps in different frames. if _, err := m.Query("db=d", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil { t.Fatal(err) @@ -144,6 +166,14 @@ func TestMain_SetProfileAttrs(t *testing.T) { m := MustRunMain() defer m.Close() + // Create frames. + client := m.Client() + if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + t.Fatal(err) + } else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + // Set bits on bitmap. if _, err := m.Query("db=d", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil { t.Fatal(err) @@ -190,6 +220,14 @@ func TestMain_FrameRestore(t *testing.T) { } m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes + // Create frames. + client := m0.Client() + if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + t.Fatal(err) + } else if err := client.CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + // Write data on first cluster. if _, err := m0.Query("db=d", ` SetBit(id=1, frame="f", profileID=100) @@ -218,6 +256,10 @@ func TestMain_FrameRestore(t *testing.T) { client, err := pilosa.NewClient(m2.Server.Host) if err != nil { t.Fatal(err) + } else if err := m2.Client().CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists { + t.Fatal(err) + } else if err := m2.Client().CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) } else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "d", "f"); err != nil { t.Fatal(err) } @@ -327,6 +369,15 @@ func (m *Main) Reopen() error { // URL returns the base URL string for accessing the running program. func (m *Main) URL() string { return "http://" + m.Server.Addr().String() } +// Client returns a client to connect to the program. +func (m *Main) Client() *pilosa.Client { + client, err := pilosa.NewClient(m.Server.Host) + if err != nil { + panic(err) + } + return client +} + // Query executes a query against the program through the HTTP API. func (m *Main) Query(rawQuery, query string) (string, error) { resp := MustDo("POST", m.URL()+"/query?"+rawQuery, query) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 7c25d62cb..5491dbfca 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -42,11 +42,24 @@ var ( // ErrPathRequired is returned when executing a command without a required path. ErrPathRequired = errors.New("path required") + Version string + BuildTime string ) +func init() { + if Version == "" { + Version = "v0.0.0" + } + if BuildTime == "" { + BuildTime = "not recorded" + } +} + func main() { m := NewMain() + fmt.Fprintf(m.Stderr, "Pilosactl %s, build time %s\n", Version, BuildTime) + // Parse command line arguments. if err := m.ParseFlags(os.Args[1:]); err == flag.ErrHelp { os.Exit(2) diff --git a/db.go b/db.go index 9d4c6532a..9348c97b6 100644 --- a/db.go +++ b/db.go @@ -15,6 +15,11 @@ import ( "github.com/pilosa/pilosa/internal" ) +// Default database settings. +const ( + DefaultColumnLabel = "profileID" +) + // DB represents a container for frames. type DB struct { mu sync.Mutex @@ -25,6 +30,9 @@ type DB struct { // This can be overridden by individual frames. timeQuantum TimeQuantum + // Label used for referring to columns in database. + columnLabel string + // Frames by name. frames map[string]*Frame @@ -40,7 +48,12 @@ type DB struct { } // NewDB returns a new instance of DB. -func NewDB(path, name string) *DB { +func NewDB(path, name string) (*DB, error) { + err := ValidateName(name) + if err != nil { + return nil, err + } + return &DB{ path: path, name: name, @@ -49,9 +62,11 @@ func NewDB(path, name string) *DB { profileAttrStore: NewAttrStore(filepath.Join(path, "data")), + columnLabel: DefaultColumnLabel, + stats: NopStatsClient, LogOutput: ioutil.Discard, - } + }, nil } // Name returns name of the database. @@ -63,6 +78,33 @@ func (db *DB) Path() string { return db.path } // ProfileAttrStore returns the storage for profile attributes. func (db *DB) ProfileAttrStore() *AttrStore { return db.profileAttrStore } +// SetColumnLabel sets the column label. Persists to meta file on update. +func (db *DB) SetColumnLabel(v string) error { + db.mu.Lock() + defer db.mu.Unlock() + + // Ignore if no change occurred. + if v == "" || db.columnLabel == v { + return nil + } + + // Persist meta data to disk on change. + db.columnLabel = v + if err := db.saveMeta(); err != nil { + return err + } + + return nil +} + +// ColumnLabel returns the column label. +func (db *DB) ColumnLabel() string { + db.mu.Lock() + v := db.columnLabel + db.mu.Unlock() + return v +} + // Open opens and initializes the database. func (db *DB) Open() error { // Ensure the path exists. @@ -104,7 +146,10 @@ func (db *DB) openFrames() error { continue } - fr := db.newFrame(db.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + fr, err := db.newFrame(db.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + if err != nil { + return ErrName + } if err := fr.Open(); err != nil { return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err) } @@ -123,6 +168,7 @@ func (db *DB) loadMeta() error { buf, err := ioutil.ReadFile(filepath.Join(db.path, "meta")) if os.IsNotExist(err) { db.timeQuantum = "" + db.columnLabel = DefaultColumnLabel return nil } else if err != nil { return err @@ -134,6 +180,7 @@ func (db *DB) loadMeta() error { // Copy metadata fields. db.timeQuantum = TimeQuantum(pb.TimeQuantum) + db.columnLabel = pb.ColumnLabel return nil } @@ -141,7 +188,10 @@ func (db *DB) loadMeta() error { // saveMeta writes meta data for the database. func (db *DB) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.DB{TimeQuantum: string(db.timeQuantum)}) + buf, err := proto.Marshal(&internal.DB{ + TimeQuantum: string(db.timeQuantum), + ColumnLabel: db.columnLabel, + }) if err != nil { return err } @@ -244,28 +294,52 @@ func (db *DB) Frames() []*Frame { return a } -// CreateFrameIfNotExists returns a frame in the database by name. -func (db *DB) CreateFrameIfNotExists(name string) (*Frame, error) { +// CreateFrame creates a frame. +func (db *DB) CreateFrame(name string, opt FrameOptions) (*Frame, error) { db.mu.Lock() defer db.mu.Unlock() - return db.createFrameIfNotExists(name) + + // Ensure frame doesn't already exist. + if db.frames[name] != nil { + return nil, ErrFrameExists + } + return db.createFrame(name, opt) } -func (db *DB) createFrameIfNotExists(name string) (*Frame, error) { - if name == "" { - return nil, errors.New("frame name required") - } +// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist. +func (db *DB) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) { + db.mu.Lock() + defer db.mu.Unlock() // Find frame in cache first. if f := db.frames[name]; f != nil { return f, nil } - // Initialize and open frame. - f := db.newFrame(db.FramePath(name), name) + return db.createFrame(name, opt) +} + +func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) { + + if name == "" { + return nil, errors.New("frame name required") + } + + // Initialize frame. + f, err := db.newFrame(db.FramePath(name), name) + if err != nil { + return nil, err + } + + // Open frame. if err := f.Open(); err != nil { return nil, err } + + // Update options. + f.SetRowLabel(opt.RowLabel) + + // Add to database's frame lookup. db.frames[name] = f db.stats.Count("frameN", 1) @@ -273,14 +347,14 @@ func (db *DB) createFrameIfNotExists(name string) (*Frame, error) { return f, nil } -func (db *DB) newFrame(path, name string) *Frame { +func (db *DB) newFrame(path, name string) (*Frame, error) { f, err := NewFrame(path, db.name, name) if err != nil { - return nil + return nil, err } f.LogOutput = db.LogOutput f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name)) - return f + return f, nil } // DeleteFrame removes a frame from the database. @@ -312,22 +386,13 @@ func (db *DB) DeleteFrame(name string) error { return nil } -// CreateFragmentIfNotExists returns a fragment in the database by name/slice. -func (db *DB) CreateFragmentIfNotExists(name string, slice uint64) (*Fragment, error) { - f, err := db.CreateFrameIfNotExists(name) - if err != nil { - return nil, err - } - return f.CreateFragmentIfNotExists(slice) -} - // SetBit sets a bit for a given profile & bitmap. // If a timestamp is specified then set all bits for the different quantum units. func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) { // Read frame. - f, err := db.CreateFrameIfNotExists(name) - if err != nil { - return changed, err + f := db.Frame(name) + if f == nil { + return changed, ErrFrameNotFound } // If this is a non-time bit then simply set the bit on the frame. @@ -345,8 +410,9 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha } // If a timestamp is specified then set bits across all frames for the quantum. + opt := f.Options() for _, subname := range FramesByTime(name, *t, q) { - f, err := db.CreateFrameIfNotExists(subname) + f, err := db.CreateFrameIfNotExists(subname, opt) if err != nil { return changed, err } @@ -363,9 +429,9 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha // Import bulk imports data. func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error { // Read frame. - f, err := db.CreateFrameIfNotExists(name) - if err != nil { - return err + f := db.Frame(name) + if f == nil { + return ErrFrameNotFound } // Determine quantum if timestamps are set. @@ -408,12 +474,12 @@ func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []* // Import into each fragment. for key, data := range dataByFragment { - f, err := db.CreateFragmentIfNotExists(key.Frame, key.Slice) + frag, err := f.CreateFragmentIfNotExists(key.Slice) if err != nil { return err } - if err := f.Import(data.BitmapIDs, data.ProfileIDs); err != nil { + if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil { return err } } @@ -475,6 +541,11 @@ func (db *DB) SetRemoteMaxSlice(newmax uint64) { db.remoteMaxSlice = newmax } +// DBOptions represents options to set when initializing a db. +type DBOptions struct { + ColumnLabel string `json:"columnLabel,omitempty"` +} + // hasTime returns true if a contains a non-nil time. func hasTime(a []*time.Time) bool { for _, t := range a { diff --git a/db_test.go b/db_test.go index db017dcd4..f0bdc60e2 100644 --- a/db_test.go +++ b/db_test.go @@ -15,7 +15,7 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) { defer db.Close() // Create frame. - f, err := db.CreateFrameIfNotExists("f") + f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if f == nil { @@ -23,7 +23,7 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) { } // Retrieve existing frame. - other, err := db.CreateFrameIfNotExists("f") + other, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } else if f != other { @@ -41,7 +41,7 @@ func TestDB_DeleteFrame(t *testing.T) { defer db.Close() // Create frame. - if _, err := db.CreateFrameIfNotExists("f"); err != nil { + if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -89,7 +89,11 @@ func NewDB() *DB { if err != nil { panic(err) } - return &DB{DB: pilosa.NewDB(path, "d")} + db, err := pilosa.NewDB(path, "d") + if err != nil { + panic(err) + } + return &DB{DB: db} } // MustOpenDB returns a new, opened database at a temporary path. Panic on error. @@ -109,12 +113,16 @@ func (db *DB) Close() error { // Reopen closes the database and reopens it. func (db *DB) Reopen() error { + var err error if err := db.DB.Close(); err != nil { return err } path, name := db.Path(), db.Name() - db.DB = pilosa.NewDB(path, name) + db.DB, err = pilosa.NewDB(path, name) + if err != nil { + return err + } if err := db.Open(); err != nil { return err @@ -130,3 +138,15 @@ func (db *DB) MustSetBit(name string, bitmapID, profileID uint64, t *time.Time) } return changed } + +// Ensure database can delete a frame. +func TestDB_InvalidName(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-db-") + if err != nil { + panic(err) + } + db, err := pilosa.NewDB(path, "ABC") + if db != nil { + t.Fatalf("unexpected db name %s", db) + } +} \ No newline at end of file diff --git a/executor.go b/executor.go index c6c058244..5ac546805 100644 --- a/executor.go +++ b/executor.go @@ -127,12 +127,14 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call // Attach bitmap attributes for Bitmap() calls. bm, _ := other.(*Bitmap) if c.Name == "Bitmap" { - id, _ := c.Args["id"].(uint64) frame, _ := c.Args["frame"].(string) fr := e.Index.Frame(db, frame) if fr != nil { - attrs, err := fr.BitmapAttrStore().Attrs(id) + rowLabel := fr.RowLabel() + rowID, _ := c.Args[rowLabel].(uint64) + + attrs, err := fr.BitmapAttrStore().Attrs(rowID) if err != nil { return nil, err } @@ -280,17 +282,27 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql } func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { - id, _ := c.Args["id"].(uint64) frame, _ := c.Args["frame"].(string) if frame == "" { frame = DefaultFrame } - f := e.Index.Fragment(db, frame, slice) + f := e.Index.Frame(db, frame) if f == nil { + return nil, ErrFrameNotFound + } + rowLabel := f.RowLabel() + + rowID, ok := c.Args[rowLabel].(uint64) + if !ok { + return nil, fmt.Errorf("Bitmap() field required: %s", rowLabel) + } + + frag := e.Index.Fragment(db, frame, slice) + if frag == nil { return NewBitmap(), nil } - return f.Bitmap(id), nil + return frag.Bitmap(rowID), nil } // executeIntersectSlice executes a intersect() call for a local slice. @@ -314,7 +326,21 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql. // executeRangeSlice executes a range() call for a local slice. func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) { - id, _ := c.Args["id"].(uint64) + // Parse frame, use default if unset. + frame, _ := c.Args["frame"].(string) + if frame == "" { + frame = DefaultFrame + } + + // Retrieve base frame. + f := e.Index.Frame(db, frame) + if f == nil { + return nil, ErrFrameNotFound + } + rowLabel := f.RowLabel() + + // Read row id. + rowID, _ := c.Args[rowLabel].(uint64) // Parse start time. startTimeStr, ok := c.Args["start"].(string) @@ -336,18 +362,6 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call return nil, errors.New("cannot parse Range() end time") } - // Parse frame, use default if unset. - frame, _ := c.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame - } - - // Retrieve base frame. - f := e.Index.Frame(db, frame) - if f == nil { - return &Bitmap{}, nil - } - // If no quantum exists then return an empty bitmap. q := f.TimeQuantum() if q == "" { @@ -361,7 +375,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call if f == nil { continue } - bm = bm.Union(f.Bitmap(id)) + bm = bm.Union(f.Bitmap(rowID)) } return bm, nil } @@ -430,27 +444,42 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, return false, errors.New("ClearBit() frame required") } - id, ok := c.Args["id"].(uint64) + // Lookup column label. + d := e.Index.DB(db) + if d == nil { + return false, nil + } + columnLabel := d.ColumnLabel() + + // Lookup row label. + f := e.Index.Frame(db, frame) + if f == nil { + return false, nil + } + rowLabel := f.RowLabel() + + // Read row & column ids. + rowID, ok := c.Args[rowLabel].(uint64) if !ok { - return false, errors.New("ClearBit() id required") + return false, fmt.Errorf("ClearBit() field required: %s", rowLabel) } - profileID, ok := c.Args["profileID"].(uint64) + colID, ok := c.Args[columnLabel].(uint64) if !ok { - return false, errors.New("ClearBit() profileID required") + return false, fmt.Errorf("ClearBit() field required: %s", columnLabel) } - slice := profileID / SliceWidth + slice := colID / SliceWidth ret := false for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - f := e.Index.Fragment(db, frame, slice) - if f == nil { + frag := e.Index.Fragment(db, frame, slice) + if frag == nil { return false, nil } - val, err := f.ClearBit(id, profileID) + val, err := frag.ClearBit(rowID, colID) if err != nil { return false, err } else if val { @@ -477,17 +506,32 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) { frame, ok := c.Args["frame"].(string) if !ok { - return false, errors.New("SetBit() frame required") + return false, errors.New("SetBit() field required: frame") } - id, ok := c.Args["id"].(uint64) - if !ok { - return false, errors.New("SetBit() id required") + // Retrieve frame. + d := e.Index.DB(db) + if d == nil { + return false, ErrFrameNotFound + } + f := d.Frame(frame) + if f == nil { + return false, ErrFrameNotFound } - profileID, ok := c.Args["profileID"].(uint64) + // Retrieve labels. + columnLabel := d.ColumnLabel() + rowLabel := f.RowLabel() + + // Read fields using labels. + rowID, ok := c.Args[rowLabel].(uint64) if !ok { - return false, errors.New("SetBit() profileID required") + return false, fmt.Errorf("SetBit() field required: %s", rowLabel) + } + + colID, ok := c.Args[columnLabel].(uint64) + if !ok { + return false, fmt.Errorf("SetBit() field required: %s", columnLabel) } var timestamp *time.Time @@ -500,17 +544,18 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op timestamp = &t } - slice := profileID / SliceWidth + slice := colID / SliceWidth ret := false for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - db, err := e.Index.CreateDBIfNotExists(db) - if err != nil { - return false, fmt.Errorf("db: %s", err) + d := e.Index.DB(db) + if d == nil { + return false, ErrDatabaseNotFound } - val, err := db.SetBit(frame, id, profileID, timestamp) + + val, err := d.SetBit(frame, rowID, colID, timestamp) if err != nil { return false, err } else if val { @@ -541,24 +586,26 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. return errors.New("SetBitmapAttrs() frame required") } - id, ok := c.Args["id"].(uint64) + // Retrieve frame. + frame := e.Index.Frame(db, frameName) + if frame == nil { + return ErrFrameNotFound + } + rowLabel := frame.RowLabel() + + // Parse labels. + rowID, ok := c.Args[rowLabel].(uint64) if !ok { - return errors.New("SetBitmapAttrs() id required") + return fmt.Errorf("SetBitmapAttrs() field required: %s", rowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, "id") - - // Retrieve frame. - frame, err := e.Index.CreateFrameIfNotExists(db, frameName) - if err != nil { - return err - } + delete(attrs, rowLabel) // Set attributes. - if err := frame.BitmapAttrStore().SetAttrs(id, attrs); err != nil { + if err := frame.BitmapAttrStore().SetAttrs(rowID, attrs); err != nil { return err } @@ -597,15 +644,22 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal return nil, errors.New("SetBitmapAttrs() frame required") } - id, ok := c.Args["id"].(uint64) + // Retrieve frame. + f := e.Index.Frame(db, frame) + if f == nil { + return nil, ErrFrameNotFound + } + rowLabel := f.RowLabel() + + rowID, ok := c.Args[rowLabel].(uint64) if !ok { - return nil, errors.New("SetBitmapAttrs() id required") + return nil, fmt.Errorf("SetBitmapAttrs() field required: %s", rowLabel) } // Copy args and remove reserved fields. attrs := pql.CopyArgs(c.Args) delete(attrs, "frame") - delete(attrs, "id") + delete(attrs, rowLabel) // Create frame group, if not exists. frameMap := m[frame] @@ -615,9 +669,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal } // Set or merge attributes. - attr := frameMap[id] + attr := frameMap[rowID] if attr == nil { - frameMap[id] = cloneAttrs(attrs) + frameMap[rowID] = cloneAttrs(attrs) } else { for k, v := range attrs { attr[k] = v @@ -628,9 +682,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal // Bulk insert attributes by frame. for name, frameMap := range m { // Retrieve frame. - frame, err := e.Index.CreateFrameIfNotExists(db, name) - if err != nil { - return nil, err + frame := e.Index.Frame(db, name) + if frame == nil { + return nil, ErrFrameNotFound } // Set attributes. @@ -677,9 +731,9 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql delete(attrs, "id") // Retrieve database. - d, err := e.Index.CreateDBIfNotExists(db) - if err != nil { - return err + d := e.Index.DB(db) + if d == nil { + return ErrDatabaseNotFound } // Set attributes. diff --git a/executor_test.go b/executor_test.go index 2cea37d3d..73ddab4e4 100644 --- a/executor_test.go +++ b/executor_test.go @@ -141,6 +141,14 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { idx := MustOpenIndex() defer idx.Close() + // Create frames. + db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } else if _, err := db.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := NewExecutor(idx.Index, NewCluster(1)) @@ -150,7 +158,7 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=XXX, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { @@ -253,8 +261,16 @@ func TestExecutor_Execute_Range(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - db := idx.MustCreateDBIfNotExists("d") + // Create database. + db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")) + + // Create frame. + if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + + // Set bits. db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-31 00:00")) db.MustSetBit("f", 1, 3, MustParseTimePtr("2000-01-01 00:00")) db.MustSetBit("f", 1, 4, MustParseTimePtr("2000-01-02 00:00")) @@ -370,6 +386,11 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { idx := MustOpenIndex() defer idx.Close() + // Create frame. + if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + e := NewExecutor(idx.Index, c) if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil { t.Fatal(err) @@ -409,7 +430,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // Create local executor data. idx := MustOpenIndex() defer idx.Close() - idx.CreateDBIfNotExists("d") + idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) oldQuantum := idx.DB("d").TimeQuantum() defer func() { // restore db quantum @@ -418,6 +439,11 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { // need to set the quantum otherwise SetBit fails silently idx.DB("d").SetTimeQuantum("Y") + // Create frame. + if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + e := NewExecutor(idx.Index, c) if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2, timestamp="2016-12-11T10:09:07")`), nil, nil); err != nil { t.Fatal(err) diff --git a/frame.go b/frame.go index 1971900e4..8ec419057 100644 --- a/frame.go +++ b/frame.go @@ -18,6 +18,11 @@ const ( FrameSuffixRank = ".n" ) +// Default frame settings. +const ( + DefaultRowLabel = "id" +) + // Frame represents a container for fragments. type Frame struct { mu sync.Mutex @@ -34,12 +39,15 @@ type Frame struct { stats StatsClient + // Label used for referring to a row. + rowLabel string + LogOutput io.Writer } // NewFrame returns a new instance of frame. -func NewFrame(path, db, name string) (*Frame, error) { - err := ValidateName(db) +func NewFrame(path, db, name string) (*Frame, error){ + err := ValidateName(name) if err != nil { return nil, err } @@ -54,6 +62,8 @@ func NewFrame(path, db, name string) (*Frame, error) { stats: NopStatsClient, + rowLabel: DefaultRowLabel, + LogOutput: ioutil.Discard, }, nil } @@ -84,6 +94,43 @@ func (f *Frame) MaxSlice() uint64 { return max } +// SetRowLabel sets the row labels. Persists to meta file on update. +func (f *Frame) SetRowLabel(v string) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Ignore if no change occurred. + if v == "" || f.rowLabel == v { + return nil + } + + // Persist meta data to disk on change. + f.rowLabel = v + if err := f.saveMeta(); err != nil { + return err + } + + return nil +} + +// RowLabel returns the row label. +func (f *Frame) RowLabel() string { + f.mu.Lock() + v := f.rowLabel + f.mu.Unlock() + return v +} + +// Options returns all options for this frame. +func (f *Frame) Options() FrameOptions { + f.mu.Lock() + opt := FrameOptions{ + RowLabel: f.rowLabel, + } + f.mu.Unlock() + return opt +} + // Open opens and initializes the frame. func (f *Frame) Open() error { if err := func() error { @@ -158,6 +205,7 @@ func (f *Frame) loadMeta() error { buf, err := ioutil.ReadFile(filepath.Join(f.path, "meta")) if os.IsNotExist(err) { f.timeQuantum = "" + f.rowLabel = DefaultRowLabel return nil } else if err != nil { return err @@ -169,6 +217,7 @@ func (f *Frame) loadMeta() error { // Copy metadata fields. f.timeQuantum = TimeQuantum(pb.TimeQuantum) + f.rowLabel = pb.RowLabel return nil } @@ -176,7 +225,10 @@ func (f *Frame) loadMeta() error { // saveMeta writes meta data for the frame. func (f *Frame) saveMeta() error { // Marshal metadata. - buf, err := proto.Marshal(&internal.Frame{TimeQuantum: string(f.timeQuantum)}) + buf, err := proto.Marshal(&internal.Frame{ + TimeQuantum: string(f.timeQuantum), + RowLabel: f.rowLabel, + }) if err != nil { return err } @@ -323,3 +375,8 @@ type frameInfoSlice []*FrameInfo func (p frameInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p frameInfoSlice) Len() int { return len(p) } func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } + +// FrameOptions represents options to set when initializing a frame. +type FrameOptions struct { + RowLabel string `json:"rowLabel,omitempty"` +} diff --git a/frame_test.go b/frame_test.go index 186db0c62..34f03ee15 100644 --- a/frame_test.go +++ b/frame_test.go @@ -65,8 +65,11 @@ func NewFrame() *Frame { if err != nil { panic(err) } - - return &Frame{Frame: pilosa.NewFrame(path, "d", "f")} + frame, err := pilosa.NewFrame(path, "d", "f") + if err != nil { + panic(err) + } + return &Frame{Frame: frame} } // MustOpenFrame returns a new, opened frame at a temporary path. Panic on error. @@ -86,15 +89,31 @@ func (f *Frame) Close() error { // Reopen closes the database and reopens it. func (f *Frame) Reopen() error { + var err error if err := f.Frame.Close(); err != nil { return err } path, db, name := f.Path(), f.DB(), f.Name() - f.Frame = pilosa.NewFrame(path, db, name) + f.Frame, err = pilosa.NewFrame(path, db, name) + if err != nil{ + return err + } if err := f.Open(); err != nil { return err } return nil } + +// NewFrame returns a new instance of Frame d/0. +func TestFrame_NameRestriction(t *testing.T) { + path, err := ioutil.TempDir("", "pilosa-frame-") + if err != nil { + panic(err) + } + frame, err := pilosa.NewFrame(path, "d", "ABC") + if frame != nil { + t.Fatalf("unexpected frame name %s", err) + } +} diff --git a/handler.go b/handler.go index 110b3892f..381cfeab7 100644 --- a/handler.go +++ b/handler.go @@ -108,6 +108,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } case "/db": switch r.Method { + case "POST": + h.handlePostDB(w, r) case "DELETE": h.handleDeleteDB(w, r) default: @@ -129,6 +131,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } case "/frame": switch r.Method { + case "POST": + h.handlePostFrame(w, r) case "DELETE": h.handleDeleteFrame(w, r) default: @@ -299,6 +303,38 @@ type sliceMaxResponse struct { MaxSlices map[string]uint64 `json:"MaxSlices"` } +// handlePostDB handles POST /db request. +func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req postDBRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Create database. + _, err := h.Index.CreateDB(req.DB, req.Options) + if err == ErrDatabaseExists { + http.Error(w, err.Error(), http.StatusConflict) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postDBResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type postDBRequest struct { + DB string `json:"db"` + Options DBOptions `json:"options"` +} + +type postDBResponse struct{} + // handleDeleteDB handles DELETE /db request. func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -343,9 +379,9 @@ func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Reques } // Retrieve database by name. - db, err := h.Index.CreateDBIfNotExists(req.DB) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + db := h.Index.DB(req.DB) + if db == nil { + http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) return } @@ -378,9 +414,9 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { } // Retrieve database from index. - db, err := h.Index.CreateDBIfNotExists(req.DB) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + db := h.Index.DB(req.DB) + if db == nil { + http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) return } @@ -424,6 +460,46 @@ type postDBAttrDiffResponse struct { Attrs map[uint64]map[string]interface{} `json:"attrs"` } +// handlePostFrame handles POST /frame request. +func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req postFrameRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Find database. + db := h.Index.DB(req.DB) + if db == nil { + http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) + return + } + + // Create frame. + _, err := db.CreateFrame(req.Frame, req.Options) + if err == ErrFrameExists { + http.Error(w, err.Error(), http.StatusConflict) + return + } else if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type postFrameRequest struct { + DB string `json:"db"` + Frame string `json:"frame"` + Options FrameOptions `json:"options"` +} + +type postFrameResponse struct{} + // handleDeleteFrame handles DELETE /frame request. func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -478,9 +554,9 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req } // Retrieve database by name. - f, err := h.Index.CreateFrameIfNotExists(req.DB, req.Frame) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + f := h.Index.Frame(req.DB, req.Frame) + if f == nil { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return } @@ -514,9 +590,9 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request } // Retrieve database from index. - f, err := h.Index.CreateFrameIfNotExists(req.DB, req.Frame) - if err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) + f := h.Index.Frame(req.DB, req.Frame) + if f == nil { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) return } @@ -714,10 +790,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { // Find the correct fragment. h.logger().Println("importing:", req.DB, req.Frame, req.Slice) - db, err := h.Index.CreateDBIfNotExists(req.DB) - if err != nil { - h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, err) - http.Error(w, "fragment error", http.StatusInternalServerError) + db := h.Index.DB(req.DB) + if db == nil { + h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrDatabaseNotFound.Error()) + http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound) return } @@ -725,6 +801,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { err = db.Import(req.Frame, req.BitmapIDs, req.ProfileIDs, timestamps) if err != nil { h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err) + return } // Marshal response object. @@ -847,15 +924,22 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) return } - // Retrieve fragment from index. - f, err := h.Index.CreateFragmentIfNotExists(q.Get("db"), q.Get("frame"), slice) + // Retrieve frame. + f := h.Index.Frame(q.Get("db"), q.Get("frame")) + if f == nil { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + return + } + + // Retrieve fragment from frame. + frag, err := f.CreateFragmentIfNotExists(slice) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } // Read fragment in from request body. - if _, err := f.ReadFrom(r.Body); err != nil { + if _, err := frag.ReadFrom(r.Body); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return } @@ -971,8 +1055,15 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) continue } + // Retrieve frame. + f := h.Index.Frame(db, frame) + if f == nil { + http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) + return + } + // Otherwise retrieve the local fragment. - f, err := h.Index.CreateFragmentIfNotExists(db, frame, slice) + frag, err := f.CreateFragmentIfNotExists(slice) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -990,7 +1081,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) // Restore to local frame and always close reader. if err := func() error { defer rd.Close() - if _, err := f.ReadFrom(rd); err != nil { + if _, err := frag.ReadFrom(rd); err != nil { return err } return nil diff --git a/handler_test.go b/handler_test.go index 49fa2160f..3a3ad3047 100644 --- a/handler_test.go +++ b/handler_test.go @@ -33,13 +33,17 @@ func TestHandler_NotFound(t *testing.T) { func TestHandler_Schema(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - if _, err := idx.CreateFrameIfNotExists("d0", "f1"); err != nil { + + d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) + d1 := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}) + + if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrameIfNotExists("d1", "f0"); err != nil { + if _, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } - if _, err := idx.CreateFrameIfNotExists("d0", "f0"); err != nil { + if _, err := d0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -210,7 +214,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { defer idx.Close() // Create database and set profile attributes. - db, err := idx.CreateDBIfNotExists("d") + db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) } else if err := db.ProfileAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { @@ -275,7 +279,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { defer idx.Close() // Create database and set profile attributes. - db, err := idx.CreateDBIfNotExists("d") + db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) } else if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { @@ -451,7 +455,7 @@ func TestHandler_DB_Delete(t *testing.T) { defer s.Close() // Create database. - if _, err := idx.CreateDBIfNotExists("d"); err != nil { + if _, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}); err != nil { t.Fatal(err) } @@ -481,7 +485,8 @@ func TestHandler_DB_Delete(t *testing.T) { func TestHandler_DeleteFrame(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - if _, err := idx.CreateFrameIfNotExists("d0", "f1"); err != nil { + d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) + if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -502,6 +507,7 @@ func TestHandler_DeleteFrame(t *testing.T) { func TestHandler_SetDBTimeQuantum(t *testing.T) { idx := MustOpenIndex() defer idx.Close() + idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}) h := NewHandler() h.Index = idx.Index @@ -521,6 +527,11 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) { idx := MustOpenIndex() defer idx.Close() + // Create frame. + if _, err := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + h := NewHandler() h.Index = idx.Index w := httptest.NewRecorder() @@ -544,7 +555,7 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) { defer s.Close() // Set attributes on the database. - db, err := idx.CreateDBIfNotExists("d") + db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) } @@ -593,7 +604,8 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { defer s.Close() // Set attributes on the database. - f, err := idx.CreateFrameIfNotExists("d", "f") + d := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}) + f, err := d.CreateFrameIfNotExists("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } @@ -657,6 +669,11 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { t.Fatalf("unexpected backup status code: %d", resp.StatusCode) } + // Create frame. + if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil { + t.Fatal(err) + } + // Restore backup to slice x/y/0. if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&slice=0", "application/octet-stream", resp.Body); err != nil { t.Fatal(err) diff --git a/index.go b/index.go index 12e3ca9bb..ee4710037 100644 --- a/index.go +++ b/index.go @@ -78,7 +78,10 @@ func (i *Index) Open() error { i.logger().Printf("opening database: %s", filepath.Base(fi.Name())) - db := i.newDB(i.DBPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + db, err := i.newDB(i.DBPath(filepath.Base(fi.Name())), filepath.Base(fi.Name())) + if err != nil { + return ErrName + } if err := db.Open(); err != nil { return fmt.Errorf("open db: name=%s, err=%s", db.Name(), err) } @@ -156,15 +159,33 @@ func (i *Index) DBs() []*DB { return a } -// CreateDBIfNotExists returns a database by name. -// The database is created if it does not already exist. -func (i *Index) CreateDBIfNotExists(name string) (*DB, error) { +// CreateDB creates a database. +func (i *Index) CreateDB(name string, opt DBOptions) (*DB, error) { i.mu.Lock() defer i.mu.Unlock() - return i.createDBIfNotExists(name) + + // Ensure db doesn't already exist. + if i.dbs[name] != nil { + return nil, ErrDatabaseExists + } + return i.createDB(name, opt) } -func (i *Index) createDBIfNotExists(name string) (*DB, error) { +// CreateDBIfNotExists returns a database by name. +// The database is created if it does not already exist. +func (i *Index) CreateDBIfNotExists(name string, opt DBOptions) (*DB, error) { + i.mu.Lock() + defer i.mu.Unlock() + + // Find frame in cache first. + if db := i.dbs[name]; db != nil { + return db, nil + } + + return i.createDB(name, opt) +} + +func (i *Index) createDB(name string, opt DBOptions) (*DB, error) { if name == "" { return nil, errors.New("database name required") } @@ -175,10 +196,18 @@ func (i *Index) createDBIfNotExists(name string) (*DB, error) { } // Otherwise create a new database. - db := i.newDB(i.DBPath(name), name) + db, err := i.newDB(i.DBPath(name), name) + if err != nil { + return nil, err + } + if err := db.Open(); err != nil { return nil, err } + + // Update options. + db.SetColumnLabel(opt.ColumnLabel) + i.dbs[db.Name()] = db i.Stats.Count("dbN", 1) @@ -186,11 +215,14 @@ func (i *Index) createDBIfNotExists(name string) (*DB, error) { return db, nil } -func (i *Index) newDB(path, name string) *DB { - db := NewDB(path, name) +func (i *Index) newDB(path, name string) (*DB, error) { + db, err := NewDB(path, name) + if err != nil { + return nil, err + } db.LogOutput = i.LogOutput db.stats = i.Stats.WithTags(fmt.Sprintf("db:%s", db.Name())) - return db + return db, nil } // DeleteDB removes a database from the index. @@ -231,16 +263,6 @@ func (i *Index) Frame(db, name string) *Frame { return d.Frame(name) } -// CreateFrameIfNotExists returns the frame for a database & name. -// The frame is created if it doesn't already exist. -func (i *Index) CreateFrameIfNotExists(db, name string) (*Frame, error) { - d, err := i.CreateDBIfNotExists(db) - if err != nil { - return nil, err - } - return d.CreateFrameIfNotExists(name) -} - // Fragment returns the fragment for a database, frame & slice. func (i *Index) Fragment(db, frame string, slice uint64) *Fragment { f := i.Frame(db, frame) @@ -250,16 +272,6 @@ func (i *Index) Fragment(db, frame string, slice uint64) *Fragment { return f.Fragment(slice) } -// CreateFragmentIfNotExists returns the fragment for a database, frame & slice. -// The fragment is created if it doesn't already exist. -func (i *Index) CreateFragmentIfNotExists(db, frame string, slice uint64) (*Fragment, error) { - f, err := i.CreateFrameIfNotExists(db, frame) - if err != nil { - return nil, err - } - return f.CreateFragmentIfNotExists(slice) -} - // monitorCacheFlush periodically flushes all fragment caches sequentially. // This is run in a goroutine. func (i *Index) monitorCacheFlush() { @@ -434,7 +446,9 @@ func (s *IndexSyncer) syncFrame(db, name string) error { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. m, err := client.BitmapAttrDiff(context.Background(), db, name, blks) - if err != nil { + if err == ErrFrameNotFound { + continue // frame not created remotely yet, skip + } else if err != nil { return err } else if len(m) == 0 { continue @@ -457,15 +471,21 @@ func (s *IndexSyncer) syncFrame(db, name string) error { // syncFragment synchronizes a fragment with the rest of the cluster. func (s *IndexSyncer) syncFragment(db, frame string, slice uint64) error { + // Retrieve local frame. + f := s.Index.Frame(db, frame) + if f == nil { + return ErrFrameNotFound + } + // Ensure fragment exists locally. - f, err := s.Index.CreateFragmentIfNotExists(db, frame, slice) + frag, err := f.CreateFragmentIfNotExists(slice) if err != nil { return err } // Sync fragments together. fs := FragmentSyncer{ - Fragment: f, + Fragment: frag, Host: s.Host, Cluster: s.Cluster, Closing: s.Closing, diff --git a/index_test.go b/index_test.go index 4707139d7..dace25fcc 100644 --- a/index_test.go +++ b/index_test.go @@ -72,6 +72,13 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { cluster.Nodes[0].Host = "localhost:0" cluster.Nodes[1].Host = MustParseURLHost(s.URL) + // Create frames on nodes. + for _, idx := range []*Index{idx0, idx1} { + idx.MustCreateFrameIfNotExists("d", "f") + idx.MustCreateFrameIfNotExists("d", "f0") + idx.MustCreateFrameIfNotExists("y", "z") + } + // Set data on the local index. f := idx0.MustCreateFragmentIfNotExists("d", "f", 0) if _, err := f.SetBit(0, 10); err != nil { @@ -191,8 +198,8 @@ func (i *Index) Close() error { } // MustCreateDBIfNotExists returns a given db. Panic on error. -func (i *Index) MustCreateDBIfNotExists(db string) *DB { - d, err := i.Index.CreateDBIfNotExists(db) +func (i *Index) MustCreateDBIfNotExists(db string, opt pilosa.DBOptions) *DB { + d, err := i.Index.CreateDBIfNotExists(db, opt) if err != nil { panic(err) } @@ -201,7 +208,7 @@ func (i *Index) MustCreateDBIfNotExists(db string) *DB { // MustCreateFrameIfNotExists returns a given frame. Panic on error. func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame { - f, err := i.Index.CreateFrameIfNotExists(db, frame) + f, err := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) if err != nil { panic(err) } @@ -210,9 +217,14 @@ func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame { // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. func (i *Index) MustCreateFragmentIfNotExists(db, frame string, slice uint64) *Fragment { - f, err := i.Index.CreateFragmentIfNotExists(db, frame, slice) + d := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{}) + f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{}) if err != nil { panic(err) } - return &Fragment{Fragment: f} + frag, err := f.CreateFragmentIfNotExists(slice) + if err != nil { + panic(err) + } + return &Fragment{Fragment: frag} } diff --git a/internal/internal.pb.go b/internal/internal.pb.go index 6239845ed..f0d8caac0 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -48,6 +48,7 @@ const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type DB struct { TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + ColumnLabel string `protobuf:"bytes,2,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"` } func (m *DB) Reset() { *m = DB{} } @@ -57,6 +58,7 @@ func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0 type Frame struct { TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + RowLabel string `protobuf:"bytes,2,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"` } func (m *Frame) Reset() { *m = Frame{} } @@ -323,6 +325,12 @@ func (m *DB) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } + if len(m.ColumnLabel) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.ColumnLabel))) + i += copy(dAtA[i:], m.ColumnLabel) + } return i, nil } @@ -347,6 +355,12 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } + if len(m.RowLabel) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.RowLabel))) + i += copy(dAtA[i:], m.RowLabel) + } return i, nil } @@ -1055,6 +1069,10 @@ func (m *DB) Size() (n int) { if l > 0 { n += 1 + l + sovInternal(uint64(l)) } + l = len(m.ColumnLabel) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } return n } @@ -1065,6 +1083,10 @@ func (m *Frame) Size() (n int) { if l > 0 { n += 1 + l + sovInternal(uint64(l)) } + l = len(m.RowLabel) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } return n } @@ -1425,6 +1447,35 @@ func (m *DB) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnLabel", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ColumnLabel = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipInternal(dAtA[iNdEx:]) @@ -1504,6 +1555,35 @@ func (m *Frame) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field RowLabel", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.RowLabel = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipInternal(dAtA[iNdEx:]) @@ -3831,49 +3911,51 @@ var ( func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 696 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6a, 0x14, 0x41, - 0x14, 0xb5, 0xa6, 0xbb, 0xe7, 0x71, 0x27, 0x19, 0x26, 0xc5, 0x28, 0x4d, 0x90, 0xa1, 0x29, 0x54, - 0x5a, 0xc1, 0x04, 0xe2, 0x46, 0x44, 0x10, 0x3b, 0x13, 0x71, 0x90, 0x84, 0xa4, 0x12, 0xdd, 0xb9, - 0x68, 0x93, 0x32, 0x69, 0xd2, 0x2f, 0xab, 0xab, 0xc5, 0x59, 0xba, 0x70, 0xe3, 0x17, 0x88, 0x7e, - 0x81, 0x7f, 0xe2, 0xd2, 0x4f, 0x90, 0xf8, 0x23, 0x52, 0x8f, 0x7e, 0x44, 0x31, 0x66, 0xe1, 0xae, - 0xef, 0xb9, 0x75, 0x6f, 0x9d, 0x73, 0x1f, 0xd5, 0x30, 0x8a, 0x52, 0xc1, 0x78, 0x1a, 0xc6, 0x6b, - 0x39, 0xcf, 0x44, 0x86, 0xfb, 0x95, 0x4d, 0x6e, 0x41, 0x67, 0x16, 0x60, 0x0f, 0x86, 0x07, 0x51, - 0xc2, 0xf6, 0xca, 0x30, 0x15, 0x65, 0xe2, 0x22, 0x0f, 0xf9, 0x03, 0xda, 0x86, 0xc8, 0x6d, 0x70, - 0x9e, 0xf0, 0x30, 0x61, 0x97, 0x38, 0x1a, 0x40, 0x37, 0x88, 0x44, 0x12, 0xe6, 0x18, 0x83, 0x1d, - 0x44, 0xa2, 0x70, 0x91, 0x67, 0xf9, 0x36, 0x55, 0xdf, 0xf8, 0x06, 0x38, 0x8f, 0x85, 0xe0, 0x85, - 0xdb, 0xf1, 0x2c, 0x7f, 0xb8, 0x31, 0x5a, 0xab, 0xa9, 0x49, 0x98, 0x6a, 0x27, 0x59, 0x03, 0x7b, - 0x37, 0x8c, 0x38, 0x1e, 0x83, 0xf5, 0x8c, 0x2d, 0xd4, 0x2d, 0x36, 0x95, 0x9f, 0x78, 0x02, 0xce, - 0x66, 0x56, 0xa6, 0xc2, 0xed, 0x28, 0x4c, 0x1b, 0xe4, 0x25, 0x58, 0x41, 0x24, 0xf0, 0x2a, 0xf4, - 0xf5, 0xd5, 0xf3, 0x99, 0x89, 0xa9, 0x6d, 0x7c, 0x1d, 0x06, 0xbb, 0x3c, 0x7b, 0x1d, 0xc5, 0x6c, - 0x3e, 0x33, 0xc1, 0x0d, 0x20, 0xbd, 0x52, 0x43, 0x21, 0xc2, 0x24, 0x77, 0x2d, 0x0f, 0xf9, 0x16, - 0x6d, 0x00, 0xf2, 0x08, 0x7a, 0xe6, 0x28, 0x1e, 0x41, 0xa7, 0x4e, 0xde, 0x99, 0xcf, 0x2e, 0xa9, - 0xe7, 0x23, 0x02, 0x5b, 0x7e, 0xb5, 0x05, 0x0d, 0xb4, 0x20, 0x0c, 0xf6, 0xc1, 0x22, 0x67, 0x86, - 0x92, 0xfa, 0x96, 0x45, 0xde, 0x17, 0x3c, 0x4a, 0x8f, 0x5f, 0x84, 0x71, 0xc9, 0x14, 0x9f, 0x01, - 0x6d, 0x43, 0x92, 0xef, 0xf3, 0x28, 0x15, 0xda, 0x6f, 0x6b, 0x35, 0x35, 0x20, 0xbd, 0x41, 0x96, - 0xc5, 0xda, 0xeb, 0x78, 0xc8, 0xef, 0xd3, 0x06, 0x20, 0xeb, 0xd0, 0x93, 0x5c, 0xb6, 0xc3, 0xbc, - 0x61, 0x8f, 0x2e, 0x62, 0xff, 0x19, 0xc1, 0xd2, 0x5e, 0xc9, 0xf8, 0x82, 0xb2, 0x37, 0x25, 0x2b, - 0x84, 0x2c, 0xc2, 0x2c, 0x30, 0x22, 0xe4, 0xfc, 0x4c, 0xc0, 0x51, 0x7e, 0x25, 0x62, 0x40, 0xb5, - 0x81, 0xaf, 0x41, 0x77, 0x3f, 0x8e, 0x0e, 0x59, 0xe1, 0x5a, 0x6a, 0x00, 0x8c, 0x25, 0xbb, 0x64, - 0xaa, 0x59, 0x28, 0xea, 0x7d, 0x5a, 0xdb, 0xd8, 0x85, 0x5e, 0x35, 0x5a, 0x8e, 0xca, 0x55, 0x99, - 0x32, 0x1b, 0x65, 0x49, 0x26, 0x98, 0xdb, 0x55, 0x31, 0xc6, 0x22, 0xef, 0x11, 0x2c, 0x1b, 0x72, - 0x45, 0x9e, 0xa5, 0x05, 0x93, 0x35, 0xde, 0xe2, 0xbc, 0xaa, 0xf1, 0x16, 0xe7, 0x78, 0x1d, 0x7a, - 0x94, 0x15, 0x65, 0x2c, 0xaa, 0x36, 0x5d, 0x6d, 0x84, 0x56, 0xb1, 0x65, 0x2c, 0x68, 0x75, 0x0a, - 0xdf, 0x6d, 0x51, 0xb4, 0x54, 0xc4, 0x4a, 0x13, 0x61, 0x3c, 0x0d, 0x6b, 0xf2, 0x01, 0xc1, 0xb0, - 0x95, 0x07, 0xfb, 0xd5, 0x0a, 0x28, 0x12, 0xc3, 0x8d, 0x71, 0x13, 0xac, 0x71, 0x5a, 0xad, 0xc8, - 0x12, 0xa0, 0x1d, 0xd3, 0x7a, 0xb4, 0x23, 0xdb, 0x21, 0xc7, 0xbe, 0xba, 0xb3, 0xd5, 0x0e, 0x09, - 0x53, 0xed, 0x94, 0x35, 0xda, 0x3c, 0x09, 0xd3, 0x63, 0x76, 0x64, 0xca, 0x57, 0x99, 0xe4, 0x2b, - 0x82, 0xe5, 0x79, 0x92, 0x67, 0x5c, 0x5c, 0xd0, 0x29, 0xb5, 0xc7, 0x55, 0xa7, 0xf4, 0x52, 0x4f, - 0xc0, 0x51, 0xbd, 0x51, 0x93, 0x66, 0x53, 0x6d, 0xa8, 0x29, 0x32, 0xdb, 0x23, 0x1b, 0x25, 0x5b, - 0xd8, 0x00, 0x78, 0x0a, 0x50, 0xaf, 0x4f, 0xe1, 0x3a, 0xca, 0xdd, 0x42, 0xa4, 0xbf, 0x5e, 0xa0, - 0xc2, 0xed, 0x7a, 0x96, 0x6f, 0xd1, 0x16, 0x42, 0x08, 0x8c, 0x2a, 0xaa, 0x7f, 0xeb, 0x1b, 0x39, - 0x82, 0x71, 0x10, 0x67, 0x87, 0xa7, 0xb3, 0x50, 0x84, 0xff, 0x43, 0xd1, 0x04, 0x1c, 0x95, 0xcf, - 0x6c, 0x8c, 0x36, 0xc8, 0x1e, 0xac, 0xb4, 0x6e, 0x31, 0x64, 0xce, 0x89, 0x47, 0x17, 0x8b, 0xef, - 0xfc, 0x2e, 0x9e, 0xdc, 0x04, 0x67, 0x33, 0x3c, 0x3c, 0xf9, 0x47, 0x1a, 0xf2, 0x05, 0xc1, 0xca, - 0x76, 0xf8, 0x4e, 0xef, 0x45, 0x7d, 0xf5, 0x53, 0x18, 0xd4, 0xa0, 0x59, 0xcc, 0x3b, 0xcd, 0x24, - 0xfc, 0x71, 0xbe, 0x41, 0xb6, 0x52, 0xc1, 0x17, 0xb4, 0x09, 0x5e, 0x7d, 0x08, 0xa3, 0xf3, 0x4e, - 0x59, 0xe3, 0xd3, 0xe6, 0xfd, 0x39, 0xd5, 0x0f, 0xea, 0x5b, 0xf5, 0x4e, 0x98, 0x07, 0x55, 0x19, - 0x0f, 0x3a, 0xf7, 0x51, 0x30, 0xfe, 0x76, 0x36, 0x45, 0xdf, 0xcf, 0xa6, 0xe8, 0xc7, 0xd9, 0x14, - 0x7d, 0xfa, 0x39, 0xbd, 0xf2, 0xaa, 0xab, 0x7e, 0x1f, 0xf7, 0x7e, 0x05, 0x00, 0x00, 0xff, 0xff, - 0x04, 0xa0, 0xe3, 0x0c, 0x50, 0x06, 0x00, 0x00, + // 721 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6a, 0x14, 0x4d, + 0x14, 0xfe, 0x6b, 0xba, 0x7b, 0x2e, 0x67, 0x92, 0x61, 0x52, 0xcc, 0xff, 0xd3, 0x84, 0x9f, 0x61, + 0x28, 0x14, 0x06, 0xc1, 0x04, 0xe2, 0x46, 0x44, 0x10, 0x7b, 0x66, 0x24, 0x83, 0x26, 0x24, 0x95, + 0xe8, 0xce, 0x45, 0x25, 0x29, 0x93, 0x26, 0x7d, 0x19, 0xab, 0xab, 0xd5, 0x59, 0xba, 0x70, 0xe3, + 0x13, 0x88, 0x3e, 0x81, 0x6f, 0xe2, 0xd2, 0x47, 0x90, 0xf8, 0x22, 0x52, 0x97, 0xbe, 0x44, 0x31, + 0x66, 0xe1, 0xae, 0xbf, 0xef, 0xd4, 0x39, 0xf5, 0x9d, 0x5b, 0x35, 0xf4, 0xc2, 0x44, 0x72, 0x91, + 0xb0, 0x68, 0x63, 0x21, 0x52, 0x99, 0xe2, 0x76, 0x81, 0xc9, 0x36, 0x34, 0xa6, 0x01, 0x1e, 0x41, + 0xf7, 0x30, 0x8c, 0xf9, 0x7e, 0xce, 0x12, 0x99, 0xc7, 0x3e, 0x1a, 0xa1, 0x71, 0x87, 0xd6, 0x29, + 0x75, 0x62, 0x92, 0x46, 0x79, 0x9c, 0x3c, 0x61, 0x47, 0x3c, 0xf2, 0x1b, 0xe6, 0x44, 0x8d, 0x22, + 0x33, 0xf0, 0x1e, 0x09, 0x16, 0xf3, 0x6b, 0x04, 0x5b, 0x87, 0x36, 0x4d, 0x5f, 0xd7, 0x23, 0x95, + 0x98, 0x04, 0xd0, 0x0c, 0x42, 0x19, 0xb3, 0x05, 0xc6, 0xe0, 0x06, 0xa1, 0xcc, 0x7c, 0x34, 0x72, + 0xc6, 0x2e, 0xd5, 0xdf, 0xf8, 0x06, 0x78, 0x0f, 0xa5, 0x14, 0x99, 0xdf, 0x18, 0x39, 0xe3, 0xee, + 0x56, 0x6f, 0xa3, 0x4c, 0x4c, 0xd1, 0xd4, 0x18, 0xc9, 0x06, 0xb8, 0x7b, 0x2c, 0x14, 0xb8, 0x0f, + 0xce, 0x63, 0xbe, 0xd4, 0x0a, 0x5c, 0xaa, 0x3e, 0xf1, 0x00, 0xbc, 0x49, 0x9a, 0x27, 0x52, 0x5f, + 0xeb, 0x52, 0x03, 0xc8, 0x73, 0x70, 0x82, 0x50, 0x2a, 0x59, 0xe6, 0xea, 0xf9, 0xd4, 0xfa, 0x94, + 0x18, 0xff, 0x0f, 0x9d, 0x3d, 0x91, 0xbe, 0x08, 0x23, 0x3e, 0x9f, 0x5a, 0xe7, 0x8a, 0x50, 0x56, + 0x95, 0x5f, 0x26, 0x59, 0xbc, 0xf0, 0x9d, 0x11, 0x1a, 0x3b, 0xb4, 0x22, 0xc8, 0x03, 0x68, 0xd9, + 0xa3, 0xb8, 0x07, 0x8d, 0x32, 0x78, 0x63, 0x3e, 0xbd, 0x66, 0x3e, 0xef, 0x11, 0xb8, 0xea, 0xab, + 0x9e, 0x50, 0xc7, 0x24, 0x84, 0xc1, 0x3d, 0x5c, 0x2e, 0xb8, 0x95, 0xa4, 0xbf, 0x55, 0x03, 0x0e, + 0xa4, 0x08, 0x93, 0xd3, 0x67, 0x2c, 0xca, 0xb9, 0xd6, 0xd3, 0xa1, 0x75, 0x4a, 0xe9, 0x7d, 0x1a, + 0x26, 0xd2, 0xd8, 0x5d, 0x93, 0x4d, 0x49, 0x28, 0x6b, 0x90, 0xa6, 0x91, 0xb1, 0x7a, 0x23, 0x34, + 0x6e, 0xd3, 0x8a, 0x20, 0x9b, 0xd0, 0x52, 0x5a, 0x76, 0xd8, 0xa2, 0x52, 0x8f, 0xae, 0x52, 0xff, + 0x11, 0xc1, 0xca, 0x7e, 0xce, 0xc5, 0x92, 0xf2, 0x97, 0x39, 0xcf, 0xa4, 0x2a, 0xc2, 0x34, 0xb0, + 0x49, 0xa8, 0xe9, 0x1b, 0x80, 0xa7, 0xed, 0x76, 0x16, 0x0c, 0xc0, 0xff, 0x41, 0xf3, 0x20, 0x0a, + 0x8f, 0x79, 0xe6, 0x3b, 0x7a, 0x00, 0x2c, 0x52, 0x5d, 0xb2, 0xd5, 0xcc, 0xb4, 0xf4, 0x36, 0x2d, + 0x31, 0xf6, 0xa1, 0x55, 0x8c, 0x9d, 0xa7, 0x63, 0x15, 0x50, 0x45, 0xa3, 0x3c, 0x4e, 0x25, 0xf7, + 0x9b, 0xda, 0xc7, 0x22, 0xf2, 0x16, 0xc1, 0xaa, 0x15, 0x97, 0x2d, 0xd2, 0x24, 0xe3, 0xaa, 0xc6, + 0x33, 0x21, 0x8a, 0x1a, 0xcf, 0x84, 0xc0, 0x9b, 0xd0, 0xa2, 0x3c, 0xcb, 0x23, 0x59, 0xb4, 0xe9, + 0xdf, 0x2a, 0xd1, 0xc2, 0x37, 0x8f, 0x24, 0x2d, 0x4e, 0xe1, 0xdb, 0x35, 0x89, 0x8e, 0xf6, 0x58, + 0xab, 0x3c, 0xac, 0xa5, 0x52, 0x4d, 0xde, 0x21, 0xe8, 0xd6, 0xe2, 0xe0, 0x71, 0xb1, 0x02, 0x5a, + 0x44, 0x77, 0xab, 0x5f, 0x39, 0x1b, 0x9e, 0x16, 0x2b, 0xb2, 0x02, 0x68, 0xd7, 0xb6, 0x1e, 0xed, + 0xaa, 0x76, 0xa8, 0xb1, 0x2f, 0xee, 0xac, 0xb5, 0x43, 0xd1, 0xd4, 0x18, 0x55, 0x8d, 0x26, 0x67, + 0x2c, 0x39, 0xe5, 0x27, 0xb6, 0x7c, 0x05, 0x24, 0x9f, 0x11, 0xac, 0xce, 0xe3, 0x45, 0x2a, 0xe4, + 0x15, 0x9d, 0xd2, 0x3b, 0x5e, 0x74, 0xca, 0x2c, 0xfc, 0x00, 0x3c, 0xdd, 0x1b, 0x3d, 0x69, 0x2e, + 0x35, 0x40, 0x4f, 0x91, 0xdd, 0x1e, 0xd5, 0x28, 0xd5, 0xc2, 0x8a, 0xc0, 0x43, 0x80, 0x72, 0x7d, + 0x32, 0xdf, 0xd3, 0xe6, 0x1a, 0xa3, 0xec, 0xe5, 0x02, 0x65, 0x7e, 0x73, 0xe4, 0x8c, 0x1d, 0x5a, + 0x63, 0x08, 0x81, 0x5e, 0x21, 0xf5, 0x77, 0x7d, 0x23, 0x27, 0xd0, 0x0f, 0xa2, 0xf4, 0xf8, 0x7c, + 0xca, 0x24, 0xfb, 0x1b, 0x19, 0x0d, 0xc0, 0xd3, 0xf1, 0xec, 0xc6, 0x18, 0x40, 0xf6, 0x61, 0xad, + 0x76, 0x8b, 0x15, 0x73, 0x29, 0x79, 0x74, 0x75, 0xf2, 0x8d, 0x9f, 0x93, 0x27, 0x37, 0xc1, 0x9b, + 0xb0, 0xe3, 0xb3, 0x3f, 0x84, 0x21, 0x9f, 0x10, 0xac, 0xed, 0xb0, 0x37, 0x66, 0x2f, 0xca, 0xab, + 0xb7, 0xa1, 0x53, 0x92, 0x76, 0x31, 0x6f, 0x55, 0x93, 0xf0, 0xcb, 0xf9, 0x8a, 0x99, 0x25, 0x52, + 0x2c, 0x69, 0xe5, 0xbc, 0x7e, 0x1f, 0x7a, 0x97, 0x8d, 0xaa, 0xc6, 0xe7, 0xd5, 0xfb, 0x73, 0x6e, + 0x1e, 0xd4, 0x57, 0xfa, 0x9d, 0xb0, 0x0f, 0xaa, 0x06, 0xf7, 0x1a, 0x77, 0x51, 0xd0, 0xff, 0x72, + 0x31, 0x44, 0x5f, 0x2f, 0x86, 0xe8, 0xdb, 0xc5, 0x10, 0x7d, 0xf8, 0x3e, 0xfc, 0xe7, 0xa8, 0xa9, + 0x7f, 0x3e, 0x77, 0x7e, 0x04, 0x00, 0x00, 0xff, 0xff, 0xf3, 0x47, 0xed, 0xb3, 0x8e, 0x06, 0x00, + 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index eaffe7307..8acbe0bab 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -4,10 +4,12 @@ package internal; message DB { string TimeQuantum = 1; + string ColumnLabel = 2; } message Frame { string TimeQuantum = 1; + string RowLabel = 2; } message Bitmap { diff --git a/pilosa.go b/pilosa.go index 1a8bba840..f6efb0987 100644 --- a/pilosa.go +++ b/pilosa.go @@ -7,24 +7,24 @@ import ( "regexp" ) +// System errors. var ( - // ErrHostRequired is returned when excuting a remote operation without a host. ErrHostRequired = errors.New("host required") - // ErrDatabaseRequired is returned when no database is specified. ErrDatabaseRequired = errors.New("database required") + ErrDatabaseExists = errors.New("database already exists") + ErrDatabaseNotFound = errors.New("database not found") - // ErrFrameRequired is returned when no frame is specified. ErrFrameRequired = errors.New("frame required") + ErrFrameExists = errors.New("frame already exists") + ErrFrameNotFound = errors.New("frame not found") // ErrFrameRequired is returned when no frame is specified. ErrName = errors.New("name restricted to [a-z0-9_-.]") // ErrFragmentNotFound is returned when a fragment does not exist. ErrFragmentNotFound = errors.New("fragment not found") - - // ErrQueryRequired is returned when no query is specified. - ErrQueryRequired = errors.New("query required") + ErrQueryRequired = errors.New("query required") ) // Profile represents vertical column in a database. @@ -83,9 +83,8 @@ const TimeFormat = "2006-01-02T15:04" // Restrict name using regex func ValidateName(name string) error { - expr := regexp.MustCompile(`^([a-z0-9._-]{2,64}$)`) + expr := regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`) validName := expr.FindStringSubmatchIndex(name) - if len(validName) == 0 { return ErrName } diff --git a/pilosactl/import.go b/pilosactl/import.go index 40a735c61..80e9a2973 100644 --- a/pilosactl/import.go +++ b/pilosactl/import.go @@ -15,7 +15,6 @@ import ( "time" "github.com/pilosa/pilosa" - "regexp" ) // ImportCommand represents a command for bulk importing data. diff --git a/roaring/roaring.go b/roaring/roaring.go index 44b9c9341..1cf271e89 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1364,6 +1364,7 @@ func unionArrayBitmap(a, b *container) *container { break } else if i >= len(a.array) { output.add(vb) + continue } else if eof { output.add(a.array[i]) i++ diff --git a/server.go b/server.go index b520381d7..57ec0e2b5 100644 --- a/server.go +++ b/server.go @@ -211,9 +211,9 @@ func (s *Server) monitorMaxSlices() { localdb.SetRemoteMaxSlice(newmax) } } else { - d, err := s.Index.CreateDBIfNotExists(db) - if err != nil { - s.logger().Printf("Failed to create DB locally: %s", db) + d := s.Index.DB(db) + if d == nil { + s.logger().Printf("Local DB not found: %s", db) return } oldmaxslices[db] = newmax