scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add tests

Browse source
This commit is contained in:
Linh Vo 2017-03-02 15:27:33 -06:00
commit 5aea6095f0
25 changed files with 934 additions and 241 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
default.etcd/
*.test
vendor

14
.travis.yml Normal file
View file

@ -0,0 +1,14 @@
language: go
go:
- 1.7.x
- master
addons:
apt:
sources:
- sourceline: 'ppa:masterminds/glide'
packages:
- glide
script: make test
notifications:
slack:
secure: k0m8IqQIjb8L5b1plR9+w4hGo5iWlGtrR9nxvPsHL7gShwez74dpHCmFUCjR/uJoZa5Xq6849IWmiBB71ZHE/wNVN60HpMeui9VKkYvUDVpgBuqoIo969ySPDyQo4jNsk0wRRv/dG5/wXo7ArY8qOp5UDjdVuEf4nEbDlBtKjo0A2W8qXBTdZ5EJ3VeP5nrtrT7knuWxXgeUyVmLkpnUaLgkY7icvH6evqkftfcb56n3l5ulKtlQkI/ij2XCUK+5xtNB9m0e1QbiqIFS7sjwlSly/GCstPVnTEp4oHNHE7LtS6TJ8lgRV90Qy6LPtROjbSdfVjhJ5cYKus8DdddSfcsm2qgHCHQThyYtVAfOXaFPYPb50WXEu3HOckHk1tYChfkdxwEGdn/cN8/xxa7Khd7T/+SaS1B4CPxOFp0ie+rD4lF6IhApmpr/UwtiW6srHcPB04lF5mioYWmyS2eyGKTzSZR82nifg5KYUgBL1FjywQDYb8t9QaAn/6svu08k4H29TBnYpOQWREb38pYmEnMtRqiRmbfQL6IyV8405TJkl/PZmIElX6IQGjf53M1KoFdCUuTL1we6rh8gen5zRVUwmM6BJxkVt4NJ2wGkrXg/HD3feYCMrv+DY3lfDN7tADx90lPb+ukmanB8oSIQHLuLCJ2XcFixEC3QSvVlnH0=

49
Makefile
View file

@ -1,10 +1,47 @@
.PHONY: vendor
.PHONY: glide vendor-update docker pilosa pilosactl crossbuild install
default:
GLIDE := $(shell command -v glide 2>/dev/null)
VERSION := $(shell git describe --tags)
IDENTIFIER := $(VERSION)-$(GOOS)-$(GOARCH)
CLONE_URL=github.com/pilosa/pilosa
BUILD_TIME=`date -u +%FT%T%z`
LDFLAGS=-ldflags "-X main.Version=$(VERSION) -X main.BuildTime=$(BUILD_TIME)"
vendor:
godep save ./...
cp -r $(GOPATH)/src/github.com/gogo/protobuf/proto/testdata vendor/github.com/gogo/protobuf/proto/testdata
default: test pilosa pilosactl
docker:
$(GOPATH)/bin:
mkdir $(GOPATH)/bin
glide: $(GOPATH)/bin
ifndef GLIDE
curl https://glide.sh/get | sh
endif
vendor: glide glide.yaml
glide install
glide.lock: glide glide.yaml
glide update
vendor-update: glide.lock
test: vendor
go test $(shell cd $(GOPATH)/src/$(CLONE_URL); go list ./... | grep -v vendor)
pilosa: vendor
go build $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
pilosactl: vendor
go build $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosactl
crossbuild: vendor
mkdir -p build/pilosa-$(IDENTIFIER)
make pilosa FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosa"
make pilosactl FLAGS="-o build/pilosa-$(IDENTIFIER)/pilosactl"
install: vendor
go install $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosa
go install $(LDFLAGS) $(FLAGS) $(CLONE_URL)/cmd/pilosactl
docker: vendor
docker build -t pilosa:latest .

1
README.md
View file

@ -2,6 +2,7 @@
Pilosa is a bitmap index database.
[![Build Status](https://travis-ci.com/pilosa/pilosa.svg?token=Peb4jvQ3kLbjUEhpU5aR&branch=master)](https://travis-ci.com/pilosa/pilosa)
## Getting Started

97
client.go
View file

@ -108,6 +108,51 @@ func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) {
return rsp.DBs, nil
}
// CreateDB creates a new database on the server.
func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error {
// Encode query request.
buf, err := json.Marshal(&postDBRequest{
DB: db,
Options: opt,
})
if err != nil {
return err
}
// Create URL & HTTP request.
u := url.URL{Scheme: "http", Host: c.host, Path: "/db"}
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return err
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return err
}
defer resp.Body.Close()
// Read body.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
// Handle response based on status code.
switch resp.StatusCode {
case http.StatusOK:
return nil // ok
case http.StatusConflict:
return ErrDatabaseExists
default:
return errors.New(string(body))
}
}
// FragmentNodes returns a list of nodes that own a slice.
func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([]*Node, error) {
// Execute request against the host.
@ -606,6 +651,56 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame str
return nil
}
// CreateFrame creates a new frame on the server.
func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOptions) error {
if db == "" {
return ErrDatabaseRequired
}
// Encode query request.
buf, err := json.Marshal(&postFrameRequest{
DB: db,
Frame: frame,
Options: opt,
})
if err != nil {
return err
}
// Create URL & HTTP request.
u := url.URL{Scheme: "http", Host: c.host, Path: "/frame"}
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return err
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
// Execute request against the host.
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return err
}
defer resp.Body.Close()
// Read body.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
// Handle response based on status code.
switch resp.StatusCode {
case http.StatusOK:
return nil // ok
case http.StatusConflict:
return ErrFrameExists
default:
return errors.New(string(body))
}
}
// RestoreFrame restores an entire frame from a host in another cluster.
func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error {
u := url.URL{
@ -807,6 +902,8 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At
// Return error if status is not OK.
switch resp.StatusCode {
case http.StatusOK: // ok
case http.StatusNotFound:
return nil, ErrFrameNotFound
default:
return nil, fmt.Errorf("unexpected status: code=%d", resp.StatusCode)
}

3
client_test.go
View file

@ -71,6 +71,9 @@ func TestClient_BackupRestore(t *testing.T) {
}
// Restore to a different frame.
if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
if err := c.RestoreFrom(context.Background(), &buf, "x", "y"); err != nil {
t.Fatal(err)
}

12
cmd/pilosa/main.go
View file

@ -17,13 +17,19 @@ import (
"github.com/pilosa/pilosa"
)
// Version holds the version information passed in at compile time.
var Version string
// Version and BuildTime hold the version/build time information passed in at compile time.
var (
Version string
BuildTime string
)
func init() {
if Version == "" {
Version = "v0.0.0"
}
if BuildTime == "" {
BuildTime = "not recorded"
}
rand.Seed(time.Now().UTC().UnixNano())
}
@ -36,7 +42,7 @@ const (
func main() {
m := NewMain()
m.Server.Handler.Version = Version
fmt.Fprintf(m.Stderr, "Pilosa %s\n", Version)
fmt.Fprintf(m.Stderr, "Pilosa %s, build time %s\n", Version, BuildTime)
// Parse command line arguments.
if err := m.ParseFlags(os.Args[1:]); err != nil {

51
cmd/pilosa/main_test.go
View file

@ -31,8 +31,20 @@ func TestMain_Set_Quick(t *testing.T) {
m := MustRunMain()
defer m.Close()
// Create client.
client, err := pilosa.NewClient(m.Server.Host)
if err != nil {
t.Fatal(err)
}
// Execute SetBit() commands.
for _, cmd := range cmds {
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
t.Fatal(err)
}
if err := client.CreateFrame(context.Background(), "d", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists {
t.Fatal(err)
}
if _, err := m.Query("db=d", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil {
t.Fatal(err)
}
@ -95,6 +107,16 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
m := MustRunMain()
defer m.Close()
// Create frames.
client := m.Client()
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "z", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Set bits on different bitmaps in different frames.
if _, err := m.Query("db=d", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil {
t.Fatal(err)
@ -144,6 +166,14 @@ func TestMain_SetProfileAttrs(t *testing.T) {
m := MustRunMain()
defer m.Close()
// Create frames.
client := m.Client()
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Set bits on bitmap.
if _, err := m.Query("db=d", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil {
t.Fatal(err)
@ -190,6 +220,14 @@ func TestMain_FrameRestore(t *testing.T) {
}
m1.Server.Cluster.Nodes = m0.Server.Cluster.Nodes
// Create frames.
client := m0.Client()
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first cluster.
if _, err := m0.Query("db=d", `
SetBit(id=1, frame="f", profileID=100)
@ -218,6 +256,10 @@ func TestMain_FrameRestore(t *testing.T) {
client, err := pilosa.NewClient(m2.Server.Host)
if err != nil {
t.Fatal(err)
} else if err := m2.Client().CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
t.Fatal(err)
} else if err := m2.Client().CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "d", "f"); err != nil {
t.Fatal(err)
}
@ -327,6 +369,15 @@ func (m *Main) Reopen() error {
// URL returns the base URL string for accessing the running program.
func (m *Main) URL() string { return "http://" + m.Server.Addr().String() }
// Client returns a client to connect to the program.
func (m *Main) Client() *pilosa.Client {
client, err := pilosa.NewClient(m.Server.Host)
if err != nil {
panic(err)
}
return client
}
// Query executes a query against the program through the HTTP API.
func (m *Main) Query(rawQuery, query string) (string, error) {
resp := MustDo("POST", m.URL()+"/query?"+rawQuery, query)

13
cmd/pilosactl/main.go
View file

@ -42,11 +42,24 @@ var (
// ErrPathRequired is returned when executing a command without a required path.
ErrPathRequired = errors.New("path required")
Version string
BuildTime string
)
func init() {
if Version == "" {
Version = "v0.0.0"
}
if BuildTime == "" {
BuildTime = "not recorded"
}
}
func main() {
m := NewMain()
fmt.Fprintf(m.Stderr, "Pilosactl %s, build time %s\n", Version, BuildTime)
// Parse command line arguments.
if err := m.ParseFlags(os.Args[1:]); err == flag.ErrHelp {
os.Exit(2)

139
db.go
View file

@ -15,6 +15,11 @@ import (
"github.com/pilosa/pilosa/internal"
)
// Default database settings.
const (
DefaultColumnLabel = "profileID"
)
// DB represents a container for frames.
type DB struct {
mu sync.Mutex
@ -25,6 +30,9 @@ type DB struct {
// This can be overridden by individual frames.
timeQuantum TimeQuantum
// Label used for referring to columns in database.
columnLabel string
// Frames by name.
frames map[string]*Frame
@ -40,7 +48,12 @@ type DB struct {
}
// NewDB returns a new instance of DB.
func NewDB(path, name string) *DB {
func NewDB(path, name string) (*DB, error) {
err := ValidateName(name)
if err != nil {
return nil, err
}
return &DB{
path: path,
name: name,
@ -49,9 +62,11 @@ func NewDB(path, name string) *DB {
profileAttrStore: NewAttrStore(filepath.Join(path, "data")),
columnLabel: DefaultColumnLabel,
stats: NopStatsClient,
LogOutput: ioutil.Discard,
}
}, nil
}
// Name returns name of the database.
@ -63,6 +78,33 @@ func (db *DB) Path() string { return db.path }
// ProfileAttrStore returns the storage for profile attributes.
func (db *DB) ProfileAttrStore() *AttrStore { return db.profileAttrStore }
// SetColumnLabel sets the column label. Persists to meta file on update.
func (db *DB) SetColumnLabel(v string) error {
db.mu.Lock()
defer db.mu.Unlock()
// Ignore if no change occurred.
if v == "" || db.columnLabel == v {
return nil
}
// Persist meta data to disk on change.
db.columnLabel = v
if err := db.saveMeta(); err != nil {
return err
}
return nil
}
// ColumnLabel returns the column label.
func (db *DB) ColumnLabel() string {
db.mu.Lock()
v := db.columnLabel
db.mu.Unlock()
return v
}
// Open opens and initializes the database.
func (db *DB) Open() error {
// Ensure the path exists.
@ -104,7 +146,10 @@ func (db *DB) openFrames() error {
continue
}
fr := db.newFrame(db.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
fr, err := db.newFrame(db.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err != nil {
return ErrName
}
if err := fr.Open(); err != nil {
return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err)
}
@ -123,6 +168,7 @@ func (db *DB) loadMeta() error {
buf, err := ioutil.ReadFile(filepath.Join(db.path, "meta"))
if os.IsNotExist(err) {
db.timeQuantum = ""
db.columnLabel = DefaultColumnLabel
return nil
} else if err != nil {
return err
@ -134,6 +180,7 @@ func (db *DB) loadMeta() error {
// Copy metadata fields.
db.timeQuantum = TimeQuantum(pb.TimeQuantum)
db.columnLabel = pb.ColumnLabel
return nil
}
@ -141,7 +188,10 @@ func (db *DB) loadMeta() error {
// saveMeta writes meta data for the database.
func (db *DB) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.DB{TimeQuantum: string(db.timeQuantum)})
buf, err := proto.Marshal(&internal.DB{
TimeQuantum: string(db.timeQuantum),
ColumnLabel: db.columnLabel,
})
if err != nil {
return err
}
@ -244,28 +294,52 @@ func (db *DB) Frames() []*Frame {
return a
}
// CreateFrameIfNotExists returns a frame in the database by name.
func (db *DB) CreateFrameIfNotExists(name string) (*Frame, error) {
// CreateFrame creates a frame.
func (db *DB) CreateFrame(name string, opt FrameOptions) (*Frame, error) {
db.mu.Lock()
defer db.mu.Unlock()
return db.createFrameIfNotExists(name)
// Ensure frame doesn't already exist.
if db.frames[name] != nil {
return nil, ErrFrameExists
}
return db.createFrame(name, opt)
}
func (db *DB) createFrameIfNotExists(name string) (*Frame, error) {
if name == "" {
return nil, errors.New("frame name required")
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (db *DB) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) {
db.mu.Lock()
defer db.mu.Unlock()
// Find frame in cache first.
if f := db.frames[name]; f != nil {
return f, nil
}
// Initialize and open frame.
f := db.newFrame(db.FramePath(name), name)
return db.createFrame(name, opt)
}
func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) {
if name == "" {
return nil, errors.New("frame name required")
}
// Initialize frame.
f, err := db.newFrame(db.FramePath(name), name)
if err != nil {
return nil, err
}
// Open frame.
if err := f.Open(); err != nil {
return nil, err
}
// Update options.
f.SetRowLabel(opt.RowLabel)
// Add to database's frame lookup.
db.frames[name] = f
db.stats.Count("frameN", 1)
@ -273,14 +347,14 @@ func (db *DB) createFrameIfNotExists(name string) (*Frame, error) {
return f, nil
}
func (db *DB) newFrame(path, name string) *Frame {
func (db *DB) newFrame(path, name string) (*Frame, error) {
f, err := NewFrame(path, db.name, name)
if err != nil {
return nil
return nil, err
}
f.LogOutput = db.LogOutput
f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name))
return f
return f, nil
}
// DeleteFrame removes a frame from the database.
@ -312,22 +386,13 @@ func (db *DB) DeleteFrame(name string) error {
return nil
}
// CreateFragmentIfNotExists returns a fragment in the database by name/slice.
func (db *DB) CreateFragmentIfNotExists(name string, slice uint64) (*Fragment, error) {
f, err := db.CreateFrameIfNotExists(name)
if err != nil {
return nil, err
}
return f.CreateFragmentIfNotExists(slice)
}
// SetBit sets a bit for a given profile & bitmap.
// If a timestamp is specified then set all bits for the different quantum units.
func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) {
// Read frame.
f, err := db.CreateFrameIfNotExists(name)
if err != nil {
return changed, err
f := db.Frame(name)
if f == nil {
return changed, ErrFrameNotFound
}
// If this is a non-time bit then simply set the bit on the frame.
@ -345,8 +410,9 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha
}
// If a timestamp is specified then set bits across all frames for the quantum.
opt := f.Options()
for _, subname := range FramesByTime(name, *t, q) {
f, err := db.CreateFrameIfNotExists(subname)
f, err := db.CreateFrameIfNotExists(subname, opt)
if err != nil {
return changed, err
}
@ -363,9 +429,9 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha
// Import bulk imports data.
func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error {
// Read frame.
f, err := db.CreateFrameIfNotExists(name)
if err != nil {
return err
f := db.Frame(name)
if f == nil {
return ErrFrameNotFound
}
// Determine quantum if timestamps are set.
@ -408,12 +474,12 @@ func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*
// Import into each fragment.
for key, data := range dataByFragment {
f, err := db.CreateFragmentIfNotExists(key.Frame, key.Slice)
frag, err := f.CreateFragmentIfNotExists(key.Slice)
if err != nil {
return err
}
if err := f.Import(data.BitmapIDs, data.ProfileIDs); err != nil {
if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil {
return err
}
}
@ -475,6 +541,11 @@ func (db *DB) SetRemoteMaxSlice(newmax uint64) {
db.remoteMaxSlice = newmax
}
// DBOptions represents options to set when initializing a db.
type DBOptions struct {
ColumnLabel string `json:"columnLabel,omitempty"`
}
// hasTime returns true if a contains a non-nil time.
func hasTime(a []*time.Time) bool {
for _, t := range a {

30
db_test.go
View file

@ -15,7 +15,7 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) {
defer db.Close()
// Create frame.
f, err := db.CreateFrameIfNotExists("f")
f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if f == nil {
@ -23,7 +23,7 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) {
}
// Retrieve existing frame.
other, err := db.CreateFrameIfNotExists("f")
other, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if f != other {
@ -41,7 +41,7 @@ func TestDB_DeleteFrame(t *testing.T) {
defer db.Close()
// Create frame.
if _, err := db.CreateFrameIfNotExists("f"); err != nil {
if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -89,7 +89,11 @@ func NewDB() *DB {
if err != nil {
panic(err)
}
return &DB{DB: pilosa.NewDB(path, "d")}
db, err := pilosa.NewDB(path, "d")
if err != nil {
panic(err)
}
return &DB{DB: db}
}
// MustOpenDB returns a new, opened database at a temporary path. Panic on error.
@ -109,12 +113,16 @@ func (db *DB) Close() error {
// Reopen closes the database and reopens it.
func (db *DB) Reopen() error {
var err error
if err := db.DB.Close(); err != nil {
return err
}
path, name := db.Path(), db.Name()
db.DB = pilosa.NewDB(path, name)
db.DB, err = pilosa.NewDB(path, name)
if err != nil {
return err
}
if err := db.Open(); err != nil {
return err
@ -130,3 +138,15 @@ func (db *DB) MustSetBit(name string, bitmapID, profileID uint64, t *time.Time)
}
return changed
}
// Ensure database can delete a frame.
func TestDB_InvalidName(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-db-")
if err != nil {
panic(err)
}
db, err := pilosa.NewDB(path, "ABC")
if db != nil {
t.Fatalf("unexpected db name %s", db)
}
}

172
executor.go
View file

@ -127,12 +127,14 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
// Attach bitmap attributes for Bitmap() calls.
bm, _ := other.(*Bitmap)
if c.Name == "Bitmap" {
id, _ := c.Args["id"].(uint64)
frame, _ := c.Args["frame"].(string)
fr := e.Index.Frame(db, frame)
if fr != nil {
attrs, err := fr.BitmapAttrStore().Attrs(id)
rowLabel := fr.RowLabel()
rowID, _ := c.Args[rowLabel].(uint64)
attrs, err := fr.BitmapAttrStore().Attrs(rowID)
if err != nil {
return nil, err
}
@ -280,17 +282,27 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql
}
func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
id, _ := c.Args["id"].(uint64)
frame, _ := c.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
f := e.Index.Fragment(db, frame, slice)
f := e.Index.Frame(db, frame)
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel := f.RowLabel()
rowID, ok := c.Args[rowLabel].(uint64)
if !ok {
return nil, fmt.Errorf("Bitmap() field required: %s", rowLabel)
}
frag := e.Index.Fragment(db, frame, slice)
if frag == nil {
return NewBitmap(), nil
}
return f.Bitmap(id), nil
return frag.Bitmap(rowID), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
@ -314,7 +326,21 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.
// executeRangeSlice executes a range() call for a local slice.
func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
id, _ := c.Args["id"].(uint64)
// Parse frame, use default if unset.
frame, _ := c.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
// Retrieve base frame.
f := e.Index.Frame(db, frame)
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel := f.RowLabel()
// Read row id.
rowID, _ := c.Args[rowLabel].(uint64)
// Parse start time.
startTimeStr, ok := c.Args["start"].(string)
@ -336,18 +362,6 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
return nil, errors.New("cannot parse Range() end time")
}
// Parse frame, use default if unset.
frame, _ := c.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
// Retrieve base frame.
f := e.Index.Frame(db, frame)
if f == nil {
return &Bitmap{}, nil
}
// If no quantum exists then return an empty bitmap.
q := f.TimeQuantum()
if q == "" {
@ -361,7 +375,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
if f == nil {
continue
}
bm = bm.Union(f.Bitmap(id))
bm = bm.Union(f.Bitmap(rowID))
}
return bm, nil
}
@ -430,27 +444,42 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
return false, errors.New("ClearBit() frame required")
}
id, ok := c.Args["id"].(uint64)
// Lookup column label.
d := e.Index.DB(db)
if d == nil {
return false, nil
}
columnLabel := d.ColumnLabel()
// Lookup row label.
f := e.Index.Frame(db, frame)
if f == nil {
return false, nil
}
rowLabel := f.RowLabel()
// Read row & column ids.
rowID, ok := c.Args[rowLabel].(uint64)
if !ok {
return false, errors.New("ClearBit() id required")
return false, fmt.Errorf("ClearBit() field required: %s", rowLabel)
}
profileID, ok := c.Args["profileID"].(uint64)
colID, ok := c.Args[columnLabel].(uint64)
if !ok {
return false, errors.New("ClearBit() profileID required")
return false, fmt.Errorf("ClearBit() field required: %s", columnLabel)
}
slice := profileID / SliceWidth
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(db, slice) {
// Update locally if host matches.
if node.Host == e.Host {
f := e.Index.Fragment(db, frame, slice)
if f == nil {
frag := e.Index.Fragment(db, frame, slice)
if frag == nil {
return false, nil
}
val, err := f.ClearBit(id, profileID)
val, err := frag.ClearBit(rowID, colID)
if err != nil {
return false, err
} else if val {
@ -477,17 +506,32 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) {
frame, ok := c.Args["frame"].(string)
if !ok {
return false, errors.New("SetBit() frame required")
return false, errors.New("SetBit() field required: frame")
}
id, ok := c.Args["id"].(uint64)
if !ok {
return false, errors.New("SetBit() id required")
// Retrieve frame.
d := e.Index.DB(db)
if d == nil {
return false, ErrFrameNotFound
}
f := d.Frame(frame)
if f == nil {
return false, ErrFrameNotFound
}
profileID, ok := c.Args["profileID"].(uint64)
// Retrieve labels.
columnLabel := d.ColumnLabel()
rowLabel := f.RowLabel()
// Read fields using labels.
rowID, ok := c.Args[rowLabel].(uint64)
if !ok {
return false, errors.New("SetBit() profileID required")
return false, fmt.Errorf("SetBit() field required: %s", rowLabel)
}
colID, ok := c.Args[columnLabel].(uint64)
if !ok {
return false, fmt.Errorf("SetBit() field required: %s", columnLabel)
}
var timestamp *time.Time
@ -500,17 +544,18 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
timestamp = &t
}
slice := profileID / SliceWidth
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(db, slice) {
// Update locally if host matches.
if node.Host == e.Host {
db, err := e.Index.CreateDBIfNotExists(db)
if err != nil {
return false, fmt.Errorf("db: %s", err)
d := e.Index.DB(db)
if d == nil {
return false, ErrDatabaseNotFound
}
val, err := db.SetBit(frame, id, profileID, timestamp)
val, err := d.SetBit(frame, rowID, colID, timestamp)
if err != nil {
return false, err
} else if val {
@ -541,24 +586,26 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
return errors.New("SetBitmapAttrs() frame required")
}
id, ok := c.Args["id"].(uint64)
// Retrieve frame.
frame := e.Index.Frame(db, frameName)
if frame == nil {
return ErrFrameNotFound
}
rowLabel := frame.RowLabel()
// Parse labels.
rowID, ok := c.Args[rowLabel].(uint64)
if !ok {
return errors.New("SetBitmapAttrs() id required")
return fmt.Errorf("SetBitmapAttrs() field required: %s", rowLabel)
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "frame")
delete(attrs, "id")
// Retrieve frame.
frame, err := e.Index.CreateFrameIfNotExists(db, frameName)
if err != nil {
return err
}
delete(attrs, rowLabel)
// Set attributes.
if err := frame.BitmapAttrStore().SetAttrs(id, attrs); err != nil {
if err := frame.BitmapAttrStore().SetAttrs(rowID, attrs); err != nil {
return err
}
@ -597,15 +644,22 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
return nil, errors.New("SetBitmapAttrs() frame required")
}
id, ok := c.Args["id"].(uint64)
// Retrieve frame.
f := e.Index.Frame(db, frame)
if f == nil {
return nil, ErrFrameNotFound
}
rowLabel := f.RowLabel()
rowID, ok := c.Args[rowLabel].(uint64)
if !ok {
return nil, errors.New("SetBitmapAttrs() id required")
return nil, fmt.Errorf("SetBitmapAttrs() field required: %s", rowLabel)
}
// Copy args and remove reserved fields.
attrs := pql.CopyArgs(c.Args)
delete(attrs, "frame")
delete(attrs, "id")
delete(attrs, rowLabel)
// Create frame group, if not exists.
frameMap := m[frame]
@ -615,9 +669,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
}
// Set or merge attributes.
attr := frameMap[id]
attr := frameMap[rowID]
if attr == nil {
frameMap[id] = cloneAttrs(attrs)
frameMap[rowID] = cloneAttrs(attrs)
} else {
for k, v := range attrs {
attr[k] = v
@ -628,9 +682,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
// Bulk insert attributes by frame.
for name, frameMap := range m {
// Retrieve frame.
frame, err := e.Index.CreateFrameIfNotExists(db, name)
if err != nil {
return nil, err
frame := e.Index.Frame(db, name)
if frame == nil {
return nil, ErrFrameNotFound
}
// Set attributes.
@ -677,9 +731,9 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql
delete(attrs, "id")
// Retrieve database.
d, err := e.Index.CreateDBIfNotExists(db)
if err != nil {
return err
d := e.Index.DB(db)
if d == nil {
return ErrDatabaseNotFound
}
// Set attributes.

32
executor_test.go
View file

@ -141,6 +141,14 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
// Create frames.
db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := db.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Set two fields on f/10.
// Also set fields on other bitmaps and frames to test isolation.
e := NewExecutor(idx.Index, NewCluster(1))
@ -150,7 +158,7 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=XXX, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
@ -253,8 +261,16 @@ func TestExecutor_Execute_Range(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
db := idx.MustCreateDBIfNotExists("d")
// Create database.
db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
db.SetTimeQuantum(pilosa.TimeQuantum("YMDH"))
// Create frame.
if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Set bits.
db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-31 00:00"))
db.MustSetBit("f", 1, 3, MustParseTimePtr("2000-01-01 00:00"))
db.MustSetBit("f", 1, 4, MustParseTimePtr("2000-01-02 00:00"))
@ -370,6 +386,11 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
// Create frame.
if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
e := NewExecutor(idx.Index, c)
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil {
t.Fatal(err)
@ -409,7 +430,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Create local executor data.
idx := MustOpenIndex()
defer idx.Close()
idx.CreateDBIfNotExists("d")
idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
oldQuantum := idx.DB("d").TimeQuantum()
defer func() {
// restore db quantum
@ -418,6 +439,11 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// need to set the quantum otherwise SetBit fails silently
idx.DB("d").SetTimeQuantum("Y")
// Create frame.
if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
e := NewExecutor(idx.Index, c)
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2, timestamp="2016-12-11T10:09:07")`), nil, nil); err != nil {
t.Fatal(err)

63
frame.go
View file

@ -18,6 +18,11 @@ const (
FrameSuffixRank = ".n"
)
// Default frame settings.
const (
DefaultRowLabel = "id"
)
// Frame represents a container for fragments.
type Frame struct {
mu sync.Mutex
@ -34,12 +39,15 @@ type Frame struct {
stats StatsClient
// Label used for referring to a row.
rowLabel string
LogOutput io.Writer
}
// NewFrame returns a new instance of frame.
func NewFrame(path, db, name string) (*Frame, error) {
err := ValidateName(db)
func NewFrame(path, db, name string) (*Frame, error){
err := ValidateName(name)
if err != nil {
return nil, err
}
@ -54,6 +62,8 @@ func NewFrame(path, db, name string) (*Frame, error) {
stats: NopStatsClient,
rowLabel: DefaultRowLabel,
LogOutput: ioutil.Discard,
}, nil
}
@ -84,6 +94,43 @@ func (f *Frame) MaxSlice() uint64 {
return max
}
// SetRowLabel sets the row labels. Persists to meta file on update.
func (f *Frame) SetRowLabel(v string) error {
f.mu.Lock()
defer f.mu.Unlock()
// Ignore if no change occurred.
if v == "" || f.rowLabel == v {
return nil
}
// Persist meta data to disk on change.
f.rowLabel = v
if err := f.saveMeta(); err != nil {
return err
}
return nil
}
// RowLabel returns the row label.
func (f *Frame) RowLabel() string {
f.mu.Lock()
v := f.rowLabel
f.mu.Unlock()
return v
}
// Options returns all options for this frame.
func (f *Frame) Options() FrameOptions {
f.mu.Lock()
opt := FrameOptions{
RowLabel: f.rowLabel,
}
f.mu.Unlock()
return opt
}
// Open opens and initializes the frame.
func (f *Frame) Open() error {
if err := func() error {
@ -158,6 +205,7 @@ func (f *Frame) loadMeta() error {
buf, err := ioutil.ReadFile(filepath.Join(f.path, "meta"))
if os.IsNotExist(err) {
f.timeQuantum = ""
f.rowLabel = DefaultRowLabel
return nil
} else if err != nil {
return err
@ -169,6 +217,7 @@ func (f *Frame) loadMeta() error {
// Copy metadata fields.
f.timeQuantum = TimeQuantum(pb.TimeQuantum)
f.rowLabel = pb.RowLabel
return nil
}
@ -176,7 +225,10 @@ func (f *Frame) loadMeta() error {
// saveMeta writes meta data for the frame.
func (f *Frame) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.Frame{TimeQuantum: string(f.timeQuantum)})
buf, err := proto.Marshal(&internal.Frame{
TimeQuantum: string(f.timeQuantum),
RowLabel: f.rowLabel,
})
if err != nil {
return err
}
@ -323,3 +375,8 @@ type frameInfoSlice []*FrameInfo
func (p frameInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p frameInfoSlice) Len() int { return len(p) }
func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FrameOptions represents options to set when initializing a frame.
type FrameOptions struct {
RowLabel string `json:"rowLabel,omitempty"`
}

25
frame_test.go
View file

@ -65,8 +65,11 @@ func NewFrame() *Frame {
if err != nil {
panic(err)
}
return &Frame{Frame: pilosa.NewFrame(path, "d", "f")}
frame, err := pilosa.NewFrame(path, "d", "f")
if err != nil {
panic(err)
}
return &Frame{Frame: frame}
}
// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error.
@ -86,15 +89,31 @@ func (f *Frame) Close() error {
// Reopen closes the database and reopens it.
func (f *Frame) Reopen() error {
var err error
if err := f.Frame.Close(); err != nil {
return err
}
path, db, name := f.Path(), f.DB(), f.Name()
f.Frame = pilosa.NewFrame(path, db, name)
f.Frame, err = pilosa.NewFrame(path, db, name)
if err != nil{
return err
}
if err := f.Open(); err != nil {
return err
}
return nil
}
// NewFrame returns a new instance of Frame d/0.
func TestFrame_NameRestriction(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "d", "ABC")
if frame != nil {
t.Fatalf("unexpected frame name %s", err)
}
}

133
handler.go
View file

@ -108,6 +108,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
case "/db":
switch r.Method {
case "POST":
h.handlePostDB(w, r)
case "DELETE":
h.handleDeleteDB(w, r)
default:
@ -129,6 +131,8 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
case "/frame":
switch r.Method {
case "POST":
h.handlePostFrame(w, r)
case "DELETE":
h.handleDeleteFrame(w, r)
default:
@ -299,6 +303,38 @@ type sliceMaxResponse struct {
MaxSlices map[string]uint64 `json:"MaxSlices"`
}
// handlePostDB handles POST /db request.
func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) {
// Decode request.
var req postDBRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Create database.
_, err := h.Index.CreateDB(req.DB, req.Options)
if err == ErrDatabaseExists {
http.Error(w, err.Error(), http.StatusConflict)
return
} else if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(postDBResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type postDBRequest struct {
DB string `json:"db"`
Options DBOptions `json:"options"`
}
type postDBResponse struct{}
// handleDeleteDB handles DELETE /db request.
func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) {
// Decode request.
@ -343,9 +379,9 @@ func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Reques
}
// Retrieve database by name.
db, err := h.Index.CreateDBIfNotExists(req.DB)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
db := h.Index.DB(req.DB)
if db == nil {
http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
return
}
@ -378,9 +414,9 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) {
}
// Retrieve database from index.
db, err := h.Index.CreateDBIfNotExists(req.DB)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
db := h.Index.DB(req.DB)
if db == nil {
http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
return
}
@ -424,6 +460,46 @@ type postDBAttrDiffResponse struct {
Attrs map[uint64]map[string]interface{} `json:"attrs"`
}
// handlePostFrame handles POST /frame request.
func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
// Decode request.
var req postFrameRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Find database.
db := h.Index.DB(req.DB)
if db == nil {
http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
return
}
// Create frame.
_, err := db.CreateFrame(req.Frame, req.Options)
if err == ErrFrameExists {
http.Error(w, err.Error(), http.StatusConflict)
return
} else if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(postFrameResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type postFrameRequest struct {
DB string `json:"db"`
Frame string `json:"frame"`
Options FrameOptions `json:"options"`
}
type postFrameResponse struct{}
// handleDeleteFrame handles DELETE /frame request.
func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
// Decode request.
@ -478,9 +554,9 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req
}
// Retrieve database by name.
f, err := h.Index.CreateFrameIfNotExists(req.DB, req.Frame)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
f := h.Index.Frame(req.DB, req.Frame)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
@ -514,9 +590,9 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request
}
// Retrieve database from index.
f, err := h.Index.CreateFrameIfNotExists(req.DB, req.Frame)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
f := h.Index.Frame(req.DB, req.Frame)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
@ -714,10 +790,10 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
// Find the correct fragment.
h.logger().Println("importing:", req.DB, req.Frame, req.Slice)
db, err := h.Index.CreateDBIfNotExists(req.DB)
if err != nil {
h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, err)
http.Error(w, "fragment error", http.StatusInternalServerError)
db := h.Index.DB(req.DB)
if db == nil {
h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrDatabaseNotFound.Error())
http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
return
}
@ -725,6 +801,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
err = db.Import(req.Frame, req.BitmapIDs, req.ProfileIDs, timestamps)
if err != nil {
h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err)
return
}
// Marshal response object.
@ -847,15 +924,22 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request)
return
}
// Retrieve fragment from index.
f, err := h.Index.CreateFragmentIfNotExists(q.Get("db"), q.Get("frame"), slice)
// Retrieve frame.
f := h.Index.Frame(q.Get("db"), q.Get("frame"))
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
// Retrieve fragment from frame.
frag, err := f.CreateFragmentIfNotExists(slice)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Read fragment in from request body.
if _, err := f.ReadFrom(r.Body); err != nil {
if _, err := frag.ReadFrom(r.Body); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@ -971,8 +1055,15 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
continue
}
// Retrieve frame.
f := h.Index.Frame(db, frame)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
// Otherwise retrieve the local fragment.
f, err := h.Index.CreateFragmentIfNotExists(db, frame, slice)
frag, err := f.CreateFragmentIfNotExists(slice)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -990,7 +1081,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
// Restore to local frame and always close reader.
if err := func() error {
defer rd.Close()
if _, err := f.ReadFrom(rd); err != nil {
if _, err := frag.ReadFrom(rd); err != nil {
return err
}
return nil

35
handler_test.go
View file

@ -33,13 +33,17 @@ func TestHandler_NotFound(t *testing.T) {
func TestHandler_Schema(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
if _, err := idx.CreateFrameIfNotExists("d0", "f1"); err != nil {
d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
d1 := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{})
if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrameIfNotExists("d1", "f0"); err != nil {
if _, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrameIfNotExists("d0", "f0"); err != nil {
if _, err := d0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -210,7 +214,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
defer idx.Close()
// Create database and set profile attributes.
db, err := idx.CreateDBIfNotExists("d")
db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
if err != nil {
t.Fatal(err)
} else if err := db.ProfileAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
@ -275,7 +279,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
defer idx.Close()
// Create database and set profile attributes.
db, err := idx.CreateDBIfNotExists("d")
db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
if err != nil {
t.Fatal(err)
} else if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
@ -451,7 +455,7 @@ func TestHandler_DB_Delete(t *testing.T) {
defer s.Close()
// Create database.
if _, err := idx.CreateDBIfNotExists("d"); err != nil {
if _, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}); err != nil {
t.Fatal(err)
}
@ -481,7 +485,8 @@ func TestHandler_DB_Delete(t *testing.T) {
func TestHandler_DeleteFrame(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
if _, err := idx.CreateFrameIfNotExists("d0", "f1"); err != nil {
d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -502,6 +507,7 @@ func TestHandler_DeleteFrame(t *testing.T) {
func TestHandler_SetDBTimeQuantum(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
h := NewHandler()
h.Index = idx.Index
@ -521,6 +527,11 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
// Create frame.
if _, err := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
h := NewHandler()
h.Index = idx.Index
w := httptest.NewRecorder()
@ -544,7 +555,7 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) {
defer s.Close()
// Set attributes on the database.
db, err := idx.CreateDBIfNotExists("d")
db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
if err != nil {
t.Fatal(err)
}
@ -593,7 +604,8 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
defer s.Close()
// Set attributes on the database.
f, err := idx.CreateFrameIfNotExists("d", "f")
d := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
f, err := d.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
@ -657,6 +669,11 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
t.Fatalf("unexpected backup status code: %d", resp.StatusCode)
}
// Create frame.
if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Restore backup to slice x/y/0.
if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&slice=0", "application/octet-stream", resp.Body); err != nil {
t.Fatal(err)

86
index.go
View file

@ -78,7 +78,10 @@ func (i *Index) Open() error {
i.logger().Printf("opening database: %s", filepath.Base(fi.Name()))
db := i.newDB(i.DBPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
db, err := i.newDB(i.DBPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err != nil {
return ErrName
}
if err := db.Open(); err != nil {
return fmt.Errorf("open db: name=%s, err=%s", db.Name(), err)
}
@ -156,15 +159,33 @@ func (i *Index) DBs() []*DB {
return a
}
// CreateDBIfNotExists returns a database by name.
// The database is created if it does not already exist.
func (i *Index) CreateDBIfNotExists(name string) (*DB, error) {
// CreateDB creates a database.
func (i *Index) CreateDB(name string, opt DBOptions) (*DB, error) {
i.mu.Lock()
defer i.mu.Unlock()
return i.createDBIfNotExists(name)
// Ensure db doesn't already exist.
if i.dbs[name] != nil {
return nil, ErrDatabaseExists
}
return i.createDB(name, opt)
}
func (i *Index) createDBIfNotExists(name string) (*DB, error) {
// CreateDBIfNotExists returns a database by name.
// The database is created if it does not already exist.
func (i *Index) CreateDBIfNotExists(name string, opt DBOptions) (*DB, error) {
i.mu.Lock()
defer i.mu.Unlock()
// Find frame in cache first.
if db := i.dbs[name]; db != nil {
return db, nil
}
return i.createDB(name, opt)
}
func (i *Index) createDB(name string, opt DBOptions) (*DB, error) {
if name == "" {
return nil, errors.New("database name required")
}
@ -175,10 +196,18 @@ func (i *Index) createDBIfNotExists(name string) (*DB, error) {
}
// Otherwise create a new database.
db := i.newDB(i.DBPath(name), name)
db, err := i.newDB(i.DBPath(name), name)
if err != nil {
return nil, err
}
if err := db.Open(); err != nil {
return nil, err
}
// Update options.
db.SetColumnLabel(opt.ColumnLabel)
i.dbs[db.Name()] = db
i.Stats.Count("dbN", 1)
@ -186,11 +215,14 @@ func (i *Index) createDBIfNotExists(name string) (*DB, error) {
return db, nil
}
func (i *Index) newDB(path, name string) *DB {
db := NewDB(path, name)
func (i *Index) newDB(path, name string) (*DB, error) {
db, err := NewDB(path, name)
if err != nil {
return nil, err
}
db.LogOutput = i.LogOutput
db.stats = i.Stats.WithTags(fmt.Sprintf("db:%s", db.Name()))
return db
return db, nil
}
// DeleteDB removes a database from the index.
@ -231,16 +263,6 @@ func (i *Index) Frame(db, name string) *Frame {
return d.Frame(name)
}
// CreateFrameIfNotExists returns the frame for a database & name.
// The frame is created if it doesn't already exist.
func (i *Index) CreateFrameIfNotExists(db, name string) (*Frame, error) {
d, err := i.CreateDBIfNotExists(db)
if err != nil {
return nil, err
}
return d.CreateFrameIfNotExists(name)
}
// Fragment returns the fragment for a database, frame & slice.
func (i *Index) Fragment(db, frame string, slice uint64) *Fragment {
f := i.Frame(db, frame)
@ -250,16 +272,6 @@ func (i *Index) Fragment(db, frame string, slice uint64) *Fragment {
return f.Fragment(slice)
}
// CreateFragmentIfNotExists returns the fragment for a database, frame & slice.
// The fragment is created if it doesn't already exist.
func (i *Index) CreateFragmentIfNotExists(db, frame string, slice uint64) (*Fragment, error) {
f, err := i.CreateFrameIfNotExists(db, frame)
if err != nil {
return nil, err
}
return f.CreateFragmentIfNotExists(slice)
}
// monitorCacheFlush periodically flushes all fragment caches sequentially.
// This is run in a goroutine.
func (i *Index) monitorCacheFlush() {
@ -434,7 +446,9 @@ func (s *IndexSyncer) syncFrame(db, name string) error {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := client.BitmapAttrDiff(context.Background(), db, name, blks)
if err != nil {
if err == ErrFrameNotFound {
continue // frame not created remotely yet, skip
} else if err != nil {
return err
} else if len(m) == 0 {
continue
@ -457,15 +471,21 @@ func (s *IndexSyncer) syncFrame(db, name string) error {
// syncFragment synchronizes a fragment with the rest of the cluster.
func (s *IndexSyncer) syncFragment(db, frame string, slice uint64) error {
// Retrieve local frame.
f := s.Index.Frame(db, frame)
if f == nil {
return ErrFrameNotFound
}
// Ensure fragment exists locally.
f, err := s.Index.CreateFragmentIfNotExists(db, frame, slice)
frag, err := f.CreateFragmentIfNotExists(slice)
if err != nil {
return err
}
// Sync fragments together.
fs := FragmentSyncer{
Fragment: f,
Fragment: frag,
Host: s.Host,
Cluster: s.Cluster,
Closing: s.Closing,

22
index_test.go
View file

@ -72,6 +72,13 @@ func TestIndexSyncer_SyncIndex(t *testing.T) {
cluster.Nodes[0].Host = "localhost:0"
cluster.Nodes[1].Host = MustParseURLHost(s.URL)
// Create frames on nodes.
for _, idx := range []*Index{idx0, idx1} {
idx.MustCreateFrameIfNotExists("d", "f")
idx.MustCreateFrameIfNotExists("d", "f0")
idx.MustCreateFrameIfNotExists("y", "z")
}
// Set data on the local index.
f := idx0.MustCreateFragmentIfNotExists("d", "f", 0)
if _, err := f.SetBit(0, 10); err != nil {
@ -191,8 +198,8 @@ func (i *Index) Close() error {
}
// MustCreateDBIfNotExists returns a given db. Panic on error.
func (i *Index) MustCreateDBIfNotExists(db string) *DB {
d, err := i.Index.CreateDBIfNotExists(db)
func (i *Index) MustCreateDBIfNotExists(db string, opt pilosa.DBOptions) *DB {
d, err := i.Index.CreateDBIfNotExists(db, opt)
if err != nil {
panic(err)
}
@ -201,7 +208,7 @@ func (i *Index) MustCreateDBIfNotExists(db string) *DB {
// MustCreateFrameIfNotExists returns a given frame. Panic on error.
func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame {
f, err := i.Index.CreateFrameIfNotExists(db, frame)
f, err := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
if err != nil {
panic(err)
}
@ -210,9 +217,14 @@ func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame {
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
func (i *Index) MustCreateFragmentIfNotExists(db, frame string, slice uint64) *Fragment {
f, err := i.Index.CreateFragmentIfNotExists(db, frame, slice)
d := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{})
f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
if err != nil {
panic(err)
}
return &Fragment{Fragment: f}
frag, err := f.CreateFragmentIfNotExists(slice)
if err != nil {
panic(err)
}
return &Fragment{Fragment: frag}
}

172
internal/internal.pb.go
View file

@ -48,6 +48,7 @@ const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type DB struct {
TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
ColumnLabel string `protobuf:"bytes,2,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"`
}
func (m *DB) Reset() { *m = DB{} }
@ -57,6 +58,7 @@ func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0
type Frame struct {
TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
RowLabel string `protobuf:"bytes,2,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"`
}
func (m *Frame) Reset() { *m = Frame{} }
@ -323,6 +325,12 @@ func (m *DB) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum)))
i += copy(dAtA[i:], m.TimeQuantum)
}
if len(m.ColumnLabel) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintInternal(dAtA, i, uint64(len(m.ColumnLabel)))
i += copy(dAtA[i:], m.ColumnLabel)
}
return i, nil
}
@ -347,6 +355,12 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum)))
i += copy(dAtA[i:], m.TimeQuantum)
}
if len(m.RowLabel) > 0 {
dAtA[i] = 0x12
i++
i = encodeVarintInternal(dAtA, i, uint64(len(m.RowLabel)))
i += copy(dAtA[i:], m.RowLabel)
}
return i, nil
}
@ -1055,6 +1069,10 @@ func (m *DB) Size() (n int) {
if l > 0 {
n += 1 + l + sovInternal(uint64(l))
}
l = len(m.ColumnLabel)
if l > 0 {
n += 1 + l + sovInternal(uint64(l))
}
return n
}
@ -1065,6 +1083,10 @@ func (m *Frame) Size() (n int) {
if l > 0 {
n += 1 + l + sovInternal(uint64(l))
}
l = len(m.RowLabel)
if l > 0 {
n += 1 + l + sovInternal(uint64(l))
}
return n
}
@ -1425,6 +1447,35 @@ func (m *DB) Unmarshal(dAtA []byte) error {
}
m.TimeQuantum = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field ColumnLabel", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowInternal
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthInternal
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.ColumnLabel = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipInternal(dAtA[iNdEx:])
@ -1504,6 +1555,35 @@ func (m *Frame) Unmarshal(dAtA []byte) error {
}
m.TimeQuantum = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field RowLabel", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowInternal
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLen |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLen := int(stringLen)
if intStringLen < 0 {
return ErrInvalidLengthInternal
}
postIndex := iNdEx + intStringLen
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.RowLabel = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipInternal(dAtA[iNdEx:])
@ -3831,49 +3911,51 @@ var (
func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) }
var fileDescriptorInternal = []byte{
// 696 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6a, 0x14, 0x41,
0x14, 0xb5, 0xa6, 0xbb, 0xe7, 0x71, 0x27, 0x19, 0x26, 0xc5, 0x28, 0x4d, 0x90, 0xa1, 0x29, 0x54,
0x5a, 0xc1, 0x04, 0xe2, 0x46, 0x44, 0x10, 0x3b, 0x13, 0x71, 0x90, 0x84, 0xa4, 0x12, 0xdd, 0xb9,
0x68, 0x93, 0x32, 0x69, 0xd2, 0x2f, 0xab, 0xab, 0xc5, 0x59, 0xba, 0x70, 0xe3, 0x17, 0x88, 0x7e,
0x81, 0x7f, 0xe2, 0xd2, 0x4f, 0x90, 0xf8, 0x23, 0x52, 0x8f, 0x7e, 0x44, 0x31, 0x66, 0xe1, 0xae,
0xef, 0xb9, 0x75, 0x6f, 0x9d, 0x73, 0x1f, 0xd5, 0x30, 0x8a, 0x52, 0xc1, 0x78, 0x1a, 0xc6, 0x6b,
0x39, 0xcf, 0x44, 0x86, 0xfb, 0x95, 0x4d, 0x6e, 0x41, 0x67, 0x16, 0x60, 0x0f, 0x86, 0x07, 0x51,
0xc2, 0xf6, 0xca, 0x30, 0x15, 0x65, 0xe2, 0x22, 0x0f, 0xf9, 0x03, 0xda, 0x86, 0xc8, 0x6d, 0x70,
0x9e, 0xf0, 0x30, 0x61, 0x97, 0x38, 0x1a, 0x40, 0x37, 0x88, 0x44, 0x12, 0xe6, 0x18, 0x83, 0x1d,
0x44, 0xa2, 0x70, 0x91, 0x67, 0xf9, 0x36, 0x55, 0xdf, 0xf8, 0x06, 0x38, 0x8f, 0x85, 0xe0, 0x85,
0xdb, 0xf1, 0x2c, 0x7f, 0xb8, 0x31, 0x5a, 0xab, 0xa9, 0x49, 0x98, 0x6a, 0x27, 0x59, 0x03, 0x7b,
0x37, 0x8c, 0x38, 0x1e, 0x83, 0xf5, 0x8c, 0x2d, 0xd4, 0x2d, 0x36, 0x95, 0x9f, 0x78, 0x02, 0xce,
0x66, 0x56, 0xa6, 0xc2, 0xed, 0x28, 0x4c, 0x1b, 0xe4, 0x25, 0x58, 0x41, 0x24, 0xf0, 0x2a, 0xf4,
0xf5, 0xd5, 0xf3, 0x99, 0x89, 0xa9, 0x6d, 0x7c, 0x1d, 0x06, 0xbb, 0x3c, 0x7b, 0x1d, 0xc5, 0x6c,
0x3e, 0x33, 0xc1, 0x0d, 0x20, 0xbd, 0x52, 0x43, 0x21, 0xc2, 0x24, 0x77, 0x2d, 0x0f, 0xf9, 0x16,
0x6d, 0x00, 0xf2, 0x08, 0x7a, 0xe6, 0x28, 0x1e, 0x41, 0xa7, 0x4e, 0xde, 0x99, 0xcf, 0x2e, 0xa9,
0xe7, 0x23, 0x02, 0x5b, 0x7e, 0xb5, 0x05, 0x0d, 0xb4, 0x20, 0x0c, 0xf6, 0xc1, 0x22, 0x67, 0x86,
0x92, 0xfa, 0x96, 0x45, 0xde, 0x17, 0x3c, 0x4a, 0x8f, 0x5f, 0x84, 0x71, 0xc9, 0x14, 0x9f, 0x01,
0x6d, 0x43, 0x92, 0xef, 0xf3, 0x28, 0x15, 0xda, 0x6f, 0x6b, 0x35, 0x35, 0x20, 0xbd, 0x41, 0x96,
0xc5, 0xda, 0xeb, 0x78, 0xc8, 0xef, 0xd3, 0x06, 0x20, 0xeb, 0xd0, 0x93, 0x5c, 0xb6, 0xc3, 0xbc,
0x61, 0x8f, 0x2e, 0x62, 0xff, 0x19, 0xc1, 0xd2, 0x5e, 0xc9, 0xf8, 0x82, 0xb2, 0x37, 0x25, 0x2b,
0x84, 0x2c, 0xc2, 0x2c, 0x30, 0x22, 0xe4, 0xfc, 0x4c, 0xc0, 0x51, 0x7e, 0x25, 0x62, 0x40, 0xb5,
0x81, 0xaf, 0x41, 0x77, 0x3f, 0x8e, 0x0e, 0x59, 0xe1, 0x5a, 0x6a, 0x00, 0x8c, 0x25, 0xbb, 0x64,
0xaa, 0x59, 0x28, 0xea, 0x7d, 0x5a, 0xdb, 0xd8, 0x85, 0x5e, 0x35, 0x5a, 0x8e, 0xca, 0x55, 0x99,
0x32, 0x1b, 0x65, 0x49, 0x26, 0x98, 0xdb, 0x55, 0x31, 0xc6, 0x22, 0xef, 0x11, 0x2c, 0x1b, 0x72,
0x45, 0x9e, 0xa5, 0x05, 0x93, 0x35, 0xde, 0xe2, 0xbc, 0xaa, 0xf1, 0x16, 0xe7, 0x78, 0x1d, 0x7a,
0x94, 0x15, 0x65, 0x2c, 0xaa, 0x36, 0x5d, 0x6d, 0x84, 0x56, 0xb1, 0x65, 0x2c, 0x68, 0x75, 0x0a,
0xdf, 0x6d, 0x51, 0xb4, 0x54, 0xc4, 0x4a, 0x13, 0x61, 0x3c, 0x0d, 0x6b, 0xf2, 0x01, 0xc1, 0xb0,
0x95, 0x07, 0xfb, 0xd5, 0x0a, 0x28, 0x12, 0xc3, 0x8d, 0x71, 0x13, 0xac, 0x71, 0x5a, 0xad, 0xc8,
0x12, 0xa0, 0x1d, 0xd3, 0x7a, 0xb4, 0x23, 0xdb, 0x21, 0xc7, 0xbe, 0xba, 0xb3, 0xd5, 0x0e, 0x09,
0x53, 0xed, 0x94, 0x35, 0xda, 0x3c, 0x09, 0xd3, 0x63, 0x76, 0x64, 0xca, 0x57, 0x99, 0xe4, 0x2b,
0x82, 0xe5, 0x79, 0x92, 0x67, 0x5c, 0x5c, 0xd0, 0x29, 0xb5, 0xc7, 0x55, 0xa7, 0xf4, 0x52, 0x4f,
0xc0, 0x51, 0xbd, 0x51, 0x93, 0x66, 0x53, 0x6d, 0xa8, 0x29, 0x32, 0xdb, 0x23, 0x1b, 0x25, 0x5b,
0xd8, 0x00, 0x78, 0x0a, 0x50, 0xaf, 0x4f, 0xe1, 0x3a, 0xca, 0xdd, 0x42, 0xa4, 0xbf, 0x5e, 0xa0,
0xc2, 0xed, 0x7a, 0x96, 0x6f, 0xd1, 0x16, 0x42, 0x08, 0x8c, 0x2a, 0xaa, 0x7f, 0xeb, 0x1b, 0x39,
0x82, 0x71, 0x10, 0x67, 0x87, 0xa7, 0xb3, 0x50, 0x84, 0xff, 0x43, 0xd1, 0x04, 0x1c, 0x95, 0xcf,
0x6c, 0x8c, 0x36, 0xc8, 0x1e, 0xac, 0xb4, 0x6e, 0x31, 0x64, 0xce, 0x89, 0x47, 0x17, 0x8b, 0xef,
0xfc, 0x2e, 0x9e, 0xdc, 0x04, 0x67, 0x33, 0x3c, 0x3c, 0xf9, 0x47, 0x1a, 0xf2, 0x05, 0xc1, 0xca,
0x76, 0xf8, 0x4e, 0xef, 0x45, 0x7d, 0xf5, 0x53, 0x18, 0xd4, 0xa0, 0x59, 0xcc, 0x3b, 0xcd, 0x24,
0xfc, 0x71, 0xbe, 0x41, 0xb6, 0x52, 0xc1, 0x17, 0xb4, 0x09, 0x5e, 0x7d, 0x08, 0xa3, 0xf3, 0x4e,
0x59, 0xe3, 0xd3, 0xe6, 0xfd, 0x39, 0xd5, 0x0f, 0xea, 0x5b, 0xf5, 0x4e, 0x98, 0x07, 0x55, 0x19,
0x0f, 0x3a, 0xf7, 0x51, 0x30, 0xfe, 0x76, 0x36, 0x45, 0xdf, 0xcf, 0xa6, 0xe8, 0xc7, 0xd9, 0x14,
0x7d, 0xfa, 0x39, 0xbd, 0xf2, 0xaa, 0xab, 0x7e, 0x1f, 0xf7, 0x7e, 0x05, 0x00, 0x00, 0xff, 0xff,
0x04, 0xa0, 0xe3, 0x0c, 0x50, 0x06, 0x00, 0x00,
// 721 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcb, 0x6a, 0x14, 0x4d,
0x14, 0xfe, 0x6b, 0xba, 0x7b, 0x2e, 0x67, 0x92, 0x61, 0x52, 0xcc, 0xff, 0xd3, 0x84, 0x9f, 0x61,
0x28, 0x14, 0x06, 0xc1, 0x04, 0xe2, 0x46, 0x44, 0x10, 0x7b, 0x66, 0x24, 0x83, 0x26, 0x24, 0x95,
0xe8, 0xce, 0x45, 0x25, 0x29, 0x93, 0x26, 0x7d, 0x19, 0xab, 0xab, 0xd5, 0x59, 0xba, 0x70, 0xe3,
0x13, 0x88, 0x3e, 0x81, 0x6f, 0xe2, 0xd2, 0x47, 0x90, 0xf8, 0x22, 0x52, 0x97, 0xbe, 0x44, 0x31,
0x66, 0xe1, 0xae, 0xbf, 0xef, 0xd4, 0x39, 0xf5, 0x9d, 0x5b, 0x35, 0xf4, 0xc2, 0x44, 0x72, 0x91,
0xb0, 0x68, 0x63, 0x21, 0x52, 0x99, 0xe2, 0x76, 0x81, 0xc9, 0x36, 0x34, 0xa6, 0x01, 0x1e, 0x41,
0xf7, 0x30, 0x8c, 0xf9, 0x7e, 0xce, 0x12, 0x99, 0xc7, 0x3e, 0x1a, 0xa1, 0x71, 0x87, 0xd6, 0x29,
0x75, 0x62, 0x92, 0x46, 0x79, 0x9c, 0x3c, 0x61, 0x47, 0x3c, 0xf2, 0x1b, 0xe6, 0x44, 0x8d, 0x22,
0x33, 0xf0, 0x1e, 0x09, 0x16, 0xf3, 0x6b, 0x04, 0x5b, 0x87, 0x36, 0x4d, 0x5f, 0xd7, 0x23, 0x95,
0x98, 0x04, 0xd0, 0x0c, 0x42, 0x19, 0xb3, 0x05, 0xc6, 0xe0, 0x06, 0xa1, 0xcc, 0x7c, 0x34, 0x72,
0xc6, 0x2e, 0xd5, 0xdf, 0xf8, 0x06, 0x78, 0x0f, 0xa5, 0x14, 0x99, 0xdf, 0x18, 0x39, 0xe3, 0xee,
0x56, 0x6f, 0xa3, 0x4c, 0x4c, 0xd1, 0xd4, 0x18, 0xc9, 0x06, 0xb8, 0x7b, 0x2c, 0x14, 0xb8, 0x0f,
0xce, 0x63, 0xbe, 0xd4, 0x0a, 0x5c, 0xaa, 0x3e, 0xf1, 0x00, 0xbc, 0x49, 0x9a, 0x27, 0x52, 0x5f,
0xeb, 0x52, 0x03, 0xc8, 0x73, 0x70, 0x82, 0x50, 0x2a, 0x59, 0xe6, 0xea, 0xf9, 0xd4, 0xfa, 0x94,
0x18, 0xff, 0x0f, 0x9d, 0x3d, 0x91, 0xbe, 0x08, 0x23, 0x3e, 0x9f, 0x5a, 0xe7, 0x8a, 0x50, 0x56,
0x95, 0x5f, 0x26, 0x59, 0xbc, 0xf0, 0x9d, 0x11, 0x1a, 0x3b, 0xb4, 0x22, 0xc8, 0x03, 0x68, 0xd9,
0xa3, 0xb8, 0x07, 0x8d, 0x32, 0x78, 0x63, 0x3e, 0xbd, 0x66, 0x3e, 0xef, 0x11, 0xb8, 0xea, 0xab,
0x9e, 0x50, 0xc7, 0x24, 0x84, 0xc1, 0x3d, 0x5c, 0x2e, 0xb8, 0x95, 0xa4, 0xbf, 0x55, 0x03, 0x0e,
0xa4, 0x08, 0x93, 0xd3, 0x67, 0x2c, 0xca, 0xb9, 0xd6, 0xd3, 0xa1, 0x75, 0x4a, 0xe9, 0x7d, 0x1a,
0x26, 0xd2, 0xd8, 0x5d, 0x93, 0x4d, 0x49, 0x28, 0x6b, 0x90, 0xa6, 0x91, 0xb1, 0x7a, 0x23, 0x34,
0x6e, 0xd3, 0x8a, 0x20, 0x9b, 0xd0, 0x52, 0x5a, 0x76, 0xd8, 0xa2, 0x52, 0x8f, 0xae, 0x52, 0xff,
0x11, 0xc1, 0xca, 0x7e, 0xce, 0xc5, 0x92, 0xf2, 0x97, 0x39, 0xcf, 0xa4, 0x2a, 0xc2, 0x34, 0xb0,
0x49, 0xa8, 0xe9, 0x1b, 0x80, 0xa7, 0xed, 0x76, 0x16, 0x0c, 0xc0, 0xff, 0x41, 0xf3, 0x20, 0x0a,
0x8f, 0x79, 0xe6, 0x3b, 0x7a, 0x00, 0x2c, 0x52, 0x5d, 0xb2, 0xd5, 0xcc, 0xb4, 0xf4, 0x36, 0x2d,
0x31, 0xf6, 0xa1, 0x55, 0x8c, 0x9d, 0xa7, 0x63, 0x15, 0x50, 0x45, 0xa3, 0x3c, 0x4e, 0x25, 0xf7,
0x9b, 0xda, 0xc7, 0x22, 0xf2, 0x16, 0xc1, 0xaa, 0x15, 0x97, 0x2d, 0xd2, 0x24, 0xe3, 0xaa, 0xc6,
0x33, 0x21, 0x8a, 0x1a, 0xcf, 0x84, 0xc0, 0x9b, 0xd0, 0xa2, 0x3c, 0xcb, 0x23, 0x59, 0xb4, 0xe9,
0xdf, 0x2a, 0xd1, 0xc2, 0x37, 0x8f, 0x24, 0x2d, 0x4e, 0xe1, 0xdb, 0x35, 0x89, 0x8e, 0xf6, 0x58,
0xab, 0x3c, 0xac, 0xa5, 0x52, 0x4d, 0xde, 0x21, 0xe8, 0xd6, 0xe2, 0xe0, 0x71, 0xb1, 0x02, 0x5a,
0x44, 0x77, 0xab, 0x5f, 0x39, 0x1b, 0x9e, 0x16, 0x2b, 0xb2, 0x02, 0x68, 0xd7, 0xb6, 0x1e, 0xed,
0xaa, 0x76, 0xa8, 0xb1, 0x2f, 0xee, 0xac, 0xb5, 0x43, 0xd1, 0xd4, 0x18, 0x55, 0x8d, 0x26, 0x67,
0x2c, 0x39, 0xe5, 0x27, 0xb6, 0x7c, 0x05, 0x24, 0x9f, 0x11, 0xac, 0xce, 0xe3, 0x45, 0x2a, 0xe4,
0x15, 0x9d, 0xd2, 0x3b, 0x5e, 0x74, 0xca, 0x2c, 0xfc, 0x00, 0x3c, 0xdd, 0x1b, 0x3d, 0x69, 0x2e,
0x35, 0x40, 0x4f, 0x91, 0xdd, 0x1e, 0xd5, 0x28, 0xd5, 0xc2, 0x8a, 0xc0, 0x43, 0x80, 0x72, 0x7d,
0x32, 0xdf, 0xd3, 0xe6, 0x1a, 0xa3, 0xec, 0xe5, 0x02, 0x65, 0x7e, 0x73, 0xe4, 0x8c, 0x1d, 0x5a,
0x63, 0x08, 0x81, 0x5e, 0x21, 0xf5, 0x77, 0x7d, 0x23, 0x27, 0xd0, 0x0f, 0xa2, 0xf4, 0xf8, 0x7c,
0xca, 0x24, 0xfb, 0x1b, 0x19, 0x0d, 0xc0, 0xd3, 0xf1, 0xec, 0xc6, 0x18, 0x40, 0xf6, 0x61, 0xad,
0x76, 0x8b, 0x15, 0x73, 0x29, 0x79, 0x74, 0x75, 0xf2, 0x8d, 0x9f, 0x93, 0x27, 0x37, 0xc1, 0x9b,
0xb0, 0xe3, 0xb3, 0x3f, 0x84, 0x21, 0x9f, 0x10, 0xac, 0xed, 0xb0, 0x37, 0x66, 0x2f, 0xca, 0xab,
0xb7, 0xa1, 0x53, 0x92, 0x76, 0x31, 0x6f, 0x55, 0x93, 0xf0, 0xcb, 0xf9, 0x8a, 0x99, 0x25, 0x52,
0x2c, 0x69, 0xe5, 0xbc, 0x7e, 0x1f, 0x7a, 0x97, 0x8d, 0xaa, 0xc6, 0xe7, 0xd5, 0xfb, 0x73, 0x6e,
0x1e, 0xd4, 0x57, 0xfa, 0x9d, 0xb0, 0x0f, 0xaa, 0x06, 0xf7, 0x1a, 0x77, 0x51, 0xd0, 0xff, 0x72,
0x31, 0x44, 0x5f, 0x2f, 0x86, 0xe8, 0xdb, 0xc5, 0x10, 0x7d, 0xf8, 0x3e, 0xfc, 0xe7, 0xa8, 0xa9,
0x7f, 0x3e, 0x77, 0x7e, 0x04, 0x00, 0x00, 0xff, 0xff, 0xf3, 0x47, 0xed, 0xb3, 0x8e, 0x06, 0x00,
0x00,
}

2
internal/internal.proto
View file

@ -4,10 +4,12 @@ package internal;
message DB {
string TimeQuantum = 1;
string ColumnLabel = 2;
}
message Frame {
string TimeQuantum = 1;
string RowLabel = 2;
}
message Bitmap {

15
pilosa.go
View file

@ -7,24 +7,24 @@ import (
"regexp"
)
// System errors.
var (
// ErrHostRequired is returned when excuting a remote operation without a host.
ErrHostRequired = errors.New("host required")
// ErrDatabaseRequired is returned when no database is specified.
ErrDatabaseRequired = errors.New("database required")
ErrDatabaseExists = errors.New("database already exists")
ErrDatabaseNotFound = errors.New("database not found")
// ErrFrameRequired is returned when no frame is specified.
ErrFrameRequired = errors.New("frame required")
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
// ErrFrameRequired is returned when no frame is specified.
ErrName = errors.New("name restricted to [a-z0-9_-.]")
// ErrFragmentNotFound is returned when a fragment does not exist.
ErrFragmentNotFound = errors.New("fragment not found")
// ErrQueryRequired is returned when no query is specified.
ErrQueryRequired = errors.New("query required")
ErrQueryRequired = errors.New("query required")
)
// Profile represents vertical column in a database.
@ -83,9 +83,8 @@ const TimeFormat = "2006-01-02T15:04"
// Restrict name using regex
func ValidateName(name string) error {
expr := regexp.MustCompile(`^([a-z0-9._-]{2,64}$)`)
expr := regexp.MustCompile(`^([a-z0-9._-]{1,64}$)`)
validName := expr.FindStringSubmatchIndex(name)
if len(validName) == 0 {
return ErrName
}

1
pilosactl/import.go
View file

@ -15,7 +15,6 @@ import (
"time"
"github.com/pilosa/pilosa"
"regexp"
)
// ImportCommand represents a command for bulk importing data.

1
roaring/roaring.go
View file

@ -1364,6 +1364,7 @@ func unionArrayBitmap(a, b *container) *container {
break
} else if i >= len(a.array) {
output.add(vb)
continue
} else if eof {
output.add(a.array[i])
i++

6
server.go
View file

@ -211,9 +211,9 @@ func (s *Server) monitorMaxSlices() {
localdb.SetRemoteMaxSlice(newmax)
}
} else {
d, err := s.Index.CreateDBIfNotExists(db)
if err != nil {
s.logger().Printf("Failed to create DB locally: %s", db)
d := s.Index.DB(db)
if d == nil {
s.logger().Printf("Local DB not found: %s", db)
return
}
oldmaxslices[db] = newmax