Rename DB to Index

Rename `db.go` to `index.go` and `db_test.go` to `index_test.go`
This commit is contained in:
Travis 2017-04-23 21:46:49 -05:00
parent 2cacff8e09
commit 2ad322c2c7
No known key found for this signature in database
GPG key ID: 7F08008DFD9314C9
47 changed files with 1962 additions and 1964 deletions

2
NOTES
View file

@ -1,5 +1,5 @@
DB Column
Index Column
┌───────────▼────────────────────────────┐
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│

View file

@ -1,6 +1,6 @@
# pilosa
Pilosa is a bitmap index database.
Pilosa is a bitmap index.
[![Build Status](https://travis-ci.com/pilosa/pilosa.svg?token=Peb4jvQ3kLbjUEhpU5aR&branch=master)](https://travis-ci.com/pilosa/pilosa)
@ -123,60 +123,60 @@ Return the version of Pilosa:
$ curl "http://127.0.0.1:10101/version"
```
Return a list of all databases and frames in the index:
Return a list of all indexes and frames in the index:
```sh
$ curl "http://127.0.0.1:10101/schema"
```
### Database and Frame Schema
### Index and Frame Schema
Before running a query, the corresponding database and frame must be created. Note that database and frame names can contain only lower case letters, numbers, dash (`-`), underscore (`_`) and dot (`.`).
Before running a query, the corresponding index and frame must be created. Note that index and frame names can contain only lower case letters, numbers, dash (`-`), underscore (`_`) and dot (`.`).
You can create the database `sample-db` using:
You can create the index `sample-idx` using:
```sh
$ curl -XPOST "http://127.0.0.1:10101/db" \
-d '{"db": "sample-db"}'
$ curl -XPOST "http://127.0.0.1:10101/index" \
-d '{"index": "sample-idx"}'
```
Optionally, you can specify the column label on database creation:
Optionally, you can specify the column label on index creation:
```sh
$ curl -XPOST "http://127.0.0.1:10101/db" \
-d '{"db": "sample-db", "options": {"columnLabel": "user"}}'
$ curl -XPOST "http://127.0.0.1:10101/index" \
-d '{"index": "sample-idx", "options": {"columnLabel": "user"}}'
```
The frame `collaboration` may be created using the following call:
```sh
$ curl -XPOST "http://127.0.0.1:10101/frame" \
-d '{"db": "sample-db", "frame": "collaboration"}'
-d '{"index": "sample-idx", "frame": "collaboration"}'
```
It is possible to specify the frame row label on frame creation:
```sh
$ curl -XPOST "http://127.0.0.1:10101/frame" \
-d '{"db": "sample-db", "frame": "collaboration", "options": {"rowLabel": "project"}}'
-d '{"index": "sample-idx", "frame": "collaboration", "options": {"rowLabel": "project"}}'
```
### Queries
Queries to Pilosa require sending a POST request where the query itself is sent as POST data.
You specify the database on which to perform the query with a URL argument `db=database-name`.
You specify the index on which to perform the query with a URL argument `index=index-name`.
In this section, we assume both the database `sample-db` with column label `user` and the frame `collaboration` with row label `project` was created.
In this section, we assume both the index `sample-idx` with column label `user` and the frame `collaboration` with row label `project` was created.
A query sent to database `sample-db` will have the following format:
A query sent to index `sample-idx` will have the following format:
```sh
$ curl -X POST "http://127.0.0.1:10101/query?db=sample-db" -d 'Query()'
$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'Query()'
```
The `Query()` object referenced above should be made up of one or more of the query types listed below.
So for example, a SetBit() query would look like this:
```sh
$ curl -X POST "http://127.0.0.1:10101/query?db=sample-db" -d 'SetBit(project=10, frame="collaboration", user=1)'
$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'SetBit(project=10, frame="collaboration", user=1)'
```
Query results have the format `{"results":[]}`, where `results` is a list of results for each `Query()`. This
@ -184,7 +184,7 @@ means that you can provide multiple `Query()` objects with each HTTP request and
the results of all of the queries.
```sh
$ curl -X POST "http://127.0.0.1:10101/query?db=sample-db" -d 'Query() Query() Query()'
$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'Query() Query() Query()'
```
---

View file

@ -84,8 +84,8 @@ var NopBroadcastReceiver = &nopBroadcastReceiver{}
const (
MessageTypeCreateSlice = 1
MessageTypeCreateDB = 2
MessageTypeDeleteDB = 3
MessageTypeCreateIndex = 2
MessageTypeDeleteIndex = 3
MessageTypeCreateFrame = 4
MessageTypeDeleteFrame = 5
)
@ -95,10 +95,10 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
switch obj := m.(type) {
case *internal.CreateSliceMessage:
typ = MessageTypeCreateSlice
case *internal.CreateDBMessage:
typ = MessageTypeCreateDB
case *internal.DeleteDBMessage:
typ = MessageTypeDeleteDB
case *internal.CreateIndexMessage:
typ = MessageTypeCreateIndex
case *internal.DeleteIndexMessage:
typ = MessageTypeDeleteIndex
case *internal.CreateFrameMessage:
typ = MessageTypeCreateFrame
case *internal.DeleteFrameMessage:
@ -120,10 +120,10 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
switch typ {
case MessageTypeCreateSlice:
m = &internal.CreateSliceMessage{}
case MessageTypeCreateDB:
m = &internal.CreateDBMessage{}
case MessageTypeDeleteDB:
m = &internal.DeleteDBMessage{}
case MessageTypeCreateIndex:
m = &internal.CreateIndexMessage{}
case MessageTypeDeleteIndex:
m = &internal.DeleteIndexMessage{}
case MessageTypeCreateFrame:
m = &internal.CreateFrameMessage{}
case MessageTypeDeleteFrame:

View file

@ -13,12 +13,12 @@ import (
func TestMessage_Marshal(t *testing.T) {
testMessageMarshal(t, &internal.CreateSliceMessage{
DB: "d",
Index: "i",
Slice: 8,
})
testMessageMarshal(t, &internal.DeleteDBMessage{
DB: "d",
testMessageMarshal(t, &internal.DeleteIndexMessage{
Index: "i",
})
}
@ -47,8 +47,8 @@ func TestBroadcast_BroadcastReceiver(t *testing.T) {
s.BroadcastReceiver = sbr
s.BroadcastReceiver.Start(sbh)
msg := &internal.DeleteDBMessage{
DB: "d",
msg := &internal.DeleteIndexMessage{
Index: "i",
}
s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg)

156
client.go
View file

@ -45,18 +45,18 @@ func NewClient(host string) (*Client, error) {
// Host returns the host the client was initialized with.
func (c *Client) Host() string { return c.host }
// MaxSliceByDatabase returns the number of slices on a server by database.
func (c *Client) MaxSliceByDatabase(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByDatabase(ctx, false)
// MaxSliceByIndex returns the number of slices on a server by index.
func (c *Client) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByIndex(ctx, false)
}
// MaxInverseSliceByDatabase returns the number of inverse slices on a server by database.
func (c *Client) MaxInverseSliceByDatabase(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByDatabase(ctx, true)
// MaxInverseSliceByIndex returns the number of inverse slices on a server by index.
func (c *Client) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByIndex(ctx, true)
}
// maxSliceByDatabase returns the number of slices on a server by database.
func (c *Client) maxSliceByDatabase(ctx context.Context, inverse bool) (map[string]uint64, error) {
// maxSliceByIndex returns the number of slices on a server by index.
func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
// Execute request against the host.
u := url.URL{
Scheme: "http",
@ -90,8 +90,8 @@ func (c *Client) maxSliceByDatabase(ctx context.Context, inverse bool) (map[stri
return rsp.MaxSlices, nil
}
// Schema returns all database and frame schema information.
func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) {
// Schema returns all index and frame schema information.
func (c *Client) Schema(ctx context.Context) ([]*IndexInfo, error) {
// Execute request against the host.
u := url.URL{
Scheme: "http",
@ -118,13 +118,13 @@ func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) {
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, fmt.Errorf("json decode: %s", err)
}
return rsp.DBs, nil
return rsp.Indexes, nil
}
// CreateDB creates a new database on the server.
func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error {
// CreateIndex creates a new index on the server.
func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
// Encode query request.
buf, err := json.Marshal(&postDBRequest{
buf, err := json.Marshal(&postIndexRequest{
Options: opt,
})
if err != nil {
@ -132,7 +132,7 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error {
}
// Create URL & HTTP request.
u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/db/%s", db)}
u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/index/%s", index)}
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return err
@ -159,20 +159,20 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error {
case http.StatusOK:
return nil // ok
case http.StatusConflict:
return ErrDatabaseExists
return ErrIndexExists
default:
return errors.New(string(body))
}
}
// FragmentNodes returns a list of nodes that own a slice.
func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([]*Node, error) {
func (c *Client) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) {
// Execute request against the host.
u := url.URL{
Scheme: "http",
Host: c.host,
Path: "/fragment/nodes",
RawQuery: (url.Values{"db": {db}, "slice": {strconv.FormatUint(slice, 10)}}).Encode(),
RawQuery: (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode(),
}
// Build request.
@ -198,10 +198,10 @@ func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([]
return a, nil
}
// ExecuteQuery executes query against db on the server.
func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedirect bool) (result interface{}, err error) {
if db == "" {
return nil, ErrDatabaseRequired
// ExecuteQuery executes query against index on the server.
func (c *Client) ExecuteQuery(ctx context.Context, index, query string, allowRedirect bool) (result interface{}, err error) {
if index == "" {
return nil, ErrIndexRequired
} else if query == "" {
return nil, ErrQueryRequired
}
@ -219,7 +219,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire
u := url.URL{
Scheme: "http",
Host: c.host,
Path: fmt.Sprintf("/db/%s/query", db),
Path: fmt.Sprintf("/index/%s/query", index),
}
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
@ -254,14 +254,14 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire
return qresp, nil
}
// ExecutePQL executes query string against db on the server.
func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{}, error) {
// ExecutePQL executes query string against index on the server.
func (c *Client) ExecutePQL(ctx context.Context, index, query string) (interface{}, error) {
u := url.URL{
Scheme: "http",
Host: c.host,
Path: "/query",
RawQuery: url.Values{
"db": {db},
"index": {index},
}.Encode(),
}
@ -287,20 +287,20 @@ func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{},
}
// Import bulk imports bits for a single slice to a host.
func (c *Client) Import(ctx context.Context, db, frame string, slice uint64, bits []Bit) error {
if db == "" {
return ErrDatabaseRequired
func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
buf, err := MarshalImportPayload(db, frame, slice, bits)
buf, err := MarshalImportPayload(index, frame, slice, bits)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, db, slice)
nodes, err := c.FragmentNodes(ctx, index, slice)
if err != nil {
return fmt.Errorf("slice nodes: %s", err)
}
@ -315,7 +315,7 @@ func (c *Client) Import(ctx context.Context, db, frame string, slice uint64, bit
return nil
}
func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, error) {
func MarshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowIDs := Bits(bits).RowIDs()
columnIDs := Bits(bits).ColumnIDs()
@ -323,7 +323,7 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e
// Marshal bits to protobufs.
buf, err := proto.Marshal(&internal.ImportRequest{
DB: db,
Index: index,
Frame: frame,
Slice: slice,
RowIDs: rowIDs,
@ -374,15 +374,15 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error {
}
// ExportCSV bulk exports data for a single slice from a host to CSV format.
func (c *Client) ExportCSV(ctx context.Context, db, frame string, slice uint64, w io.Writer) error {
if db == "" {
return ErrDatabaseRequired
func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, db, slice)
nodes, err := c.FragmentNodes(ctx, index, slice)
if err != nil {
return fmt.Errorf("slice nodes: %s", err)
}
@ -392,7 +392,7 @@ func (c *Client) ExportCSV(ctx context.Context, db, frame string, slice uint64,
for _, i := range rand.Perm(len(nodes)) {
node := nodes[i]
if err := c.exportNodeCSV(ctx, node, db, frame, slice, w); err != nil {
if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil {
e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err)
continue
} else {
@ -404,14 +404,14 @@ func (c *Client) ExportCSV(ctx context.Context, db, frame string, slice uint64,
}
// exportNode copies a CSV export from a node to w.
func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string, slice uint64, w io.Writer) error {
func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error {
// Create URL.
u := url.URL{
Scheme: "http",
Host: node.Host,
Path: "/export",
RawQuery: url.Values{
"db": {db},
"index": {index},
"frame": {frame},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode(),
@ -445,9 +445,9 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string
}
// BackupTo backs up an entire frame from a cluster to w.
func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view string) error {
if db == "" {
return ErrDatabaseRequired
func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
@ -456,14 +456,14 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view stri
tw := tar.NewWriter(w)
// Find the maximum number of slices.
maxSlices, err := c.MaxSliceByDatabase(ctx)
maxSlices, err := c.MaxSliceByIndex(ctx)
if err != nil {
return fmt.Errorf("slice n: %s", err)
}
// Backup every slice to the tar file.
for i := uint64(0); i <= maxSlices[db]; i++ {
if err := c.backupSliceTo(ctx, tw, db, frame, view, i); err != nil {
for i := uint64(0); i <= maxSlices[index]; i++ {
if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil {
return err
}
}
@ -477,9 +477,9 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view stri
}
// backupSliceTo backs up a single slice to tw.
func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame, view string, slice uint64) error {
func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error {
// Return error if unable to backup from any slice.
r, err := c.BackupSlice(ctx, db, frame, view, slice)
r, err := c.BackupSlice(ctx, index, frame, view, slice)
if err != nil {
return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err)
} else if r == nil {
@ -515,16 +515,16 @@ func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame, v
// BackupSlice retrieves a streaming backup from a single slice.
// This function tries slice owners until one succeeds.
func (c *Client) BackupSlice(ctx context.Context, db, frame, view string, slice uint64) (io.ReadCloser, error) {
func (c *Client) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) {
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, db, slice)
nodes, err := c.FragmentNodes(ctx, index, slice)
if err != nil {
return nil, fmt.Errorf("slice nodes: %s", err)
}
// Try to backup slice from each one until successful.
for _, i := range rand.Perm(len(nodes)) {
r, err := c.backupSliceNode(ctx, db, frame, view, slice, nodes[i])
r, err := c.backupSliceNode(ctx, index, frame, view, slice, nodes[i])
if err == nil {
return r, nil // successfully attached
} else if err == ErrFragmentNotFound {
@ -538,13 +538,13 @@ func (c *Client) BackupSlice(ctx context.Context, db, frame, view string, slice
return nil, fmt.Errorf("unable to connect to any owner")
}
func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
func (c *Client) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
u := url.URL{
Scheme: "http",
Host: node.Host,
Path: "/fragment/data",
RawQuery: url.Values{
"db": {db},
"index": {index},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
@ -576,9 +576,9 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, sl
}
// RestoreFrom restores a frame from a backup file to an entire cluster.
func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame, view string) error {
if db == "" {
return ErrDatabaseRequired
func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
@ -608,16 +608,16 @@ func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame, view s
}
// Restore file to all nodes that own it.
if err := c.restoreSliceFrom(ctx, buf.Bytes(), db, frame, view, slice); err != nil {
if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil {
return err
}
}
}
// restoreSliceFrom restores a single slice to all owning nodes.
func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, view string, slice uint64) error {
func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error {
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, db, slice)
nodes, err := c.FragmentNodes(ctx, index, slice)
if err != nil {
return fmt.Errorf("slice nodes: %s", err)
}
@ -629,7 +629,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, vi
Host: node.Host,
Path: "/fragment/data",
RawQuery: url.Values{
"db": {db},
"index": {index},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
@ -659,9 +659,9 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, vi
}
// CreateFrame creates a new frame on the server.
func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOptions) error {
if db == "" {
return ErrDatabaseRequired
func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error {
if index == "" {
return ErrIndexRequired
}
// Encode query request.
@ -673,7 +673,7 @@ func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOpt
}
// Create URL & HTTP request.
u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/db/%s/frame/%s", db, frame)}
u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/index/%s/frame/%s", index, frame)}
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return err
@ -707,11 +707,11 @@ func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOpt
}
// RestoreFrame restores an entire frame from a host in another cluster.
func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error {
func (c *Client) RestoreFrame(ctx context.Context, host, index, frame string) error {
u := url.URL{
Scheme: "http",
Host: c.Host(),
Path: fmt.Sprintf("/db/%s/frame/%s/restore", db, frame),
Path: fmt.Sprintf("/index/%s/frame/%s/restore", index, frame),
RawQuery: url.Values{
"host": {host},
}.Encode(),
@ -740,12 +740,12 @@ func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error
}
// FrameViews returns a list of view names for a frame.
func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, error) {
func (c *Client) FrameViews(ctx context.Context, index, frame string) ([]string, error) {
// Create URL & HTTP request.
u := url.URL{
Scheme: "http",
Host: c.host,
Path: fmt.Sprintf("/db/%s/frame/%s/views", db, frame),
Path: fmt.Sprintf("/index/%s/frame/%s/views", index, frame),
}
req, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
@ -780,13 +780,13 @@ func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, er
// FragmentBlocks returns a list of block checksums for a fragment on a host.
// Only returns blocks which contain data.
func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, slice uint64) ([]FragmentBlock, error) {
func (c *Client) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) {
u := url.URL{
Scheme: "http",
Host: c.host,
Path: "/fragment/blocks",
RawQuery: url.Values{
"db": {db},
"index": {index},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
@ -824,9 +824,9 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, sli
}
// BlockData returns row/column id pairs for a block.
func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) {
func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) {
buf, err := proto.Marshal(&internal.BlockDataRequest{
DB: db,
Index: index,
Frame: frame,
View: view,
Slice: slice,
@ -871,15 +871,15 @@ func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice ui
}
// ColumnAttrDiff returns data from differing blocks on a remote host.
func (c *Client) ColumnAttrDiff(ctx context.Context, db string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
func (c *Client) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
u := url.URL{
Scheme: "http",
Host: c.host,
Path: fmt.Sprintf("/db/%s/attr/diff", db),
Path: fmt.Sprintf("/index/%s/attr/diff", index),
}
// Encode request.
buf, err := json.Marshal(postDBAttrDiffRequest{Blocks: blks})
buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks})
if err != nil {
return nil, err
}
@ -906,7 +906,7 @@ func (c *Client) ColumnAttrDiff(ctx context.Context, db string, blks []AttrBlock
}
// Decode response object.
var rsp postDBAttrDiffResponse
var rsp postIndexAttrDiffResponse
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
return nil, err
}
@ -914,11 +914,11 @@ func (c *Client) ColumnAttrDiff(ctx context.Context, db string, blks []AttrBlock
}
// RowAttrDiff returns data from differing blocks on a remote host.
func (c *Client) RowAttrDiff(ctx context.Context, db, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
func (c *Client) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
u := url.URL{
Scheme: "http",
Host: c.host,
Path: fmt.Sprintf("/db/%s/frame/%s/attr/diff", db, frame),
Path: fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame),
}
// Encode request.

View file

@ -38,56 +38,56 @@ func TestClient_MultiNode(t *testing.T) {
defer s[i].Close()
}
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor()
e.Holder = hldr[0].Holder
e.Host = cluster.Nodes[0].Host
e.Cluster = cluster
return e.Execute(ctx, db, query, slices, opt)
return e.Execute(ctx, index, query, slices, opt)
}
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor()
e.Holder = hldr[1].Holder
e.Host = cluster.Nodes[1].Host
e.Cluster = cluster
return e.Execute(ctx, db, query, slices, opt)
return e.Execute(ctx, index, query, slices, opt)
}
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor()
e.Holder = hldr[2].Holder
e.Host = cluster.Nodes[2].Host
e.Cluster = cluster
return e.Execute(ctx, db, query, slices, opt)
return e.Execute(ctx, index, query, slices, opt)
}
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4)
hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(1, 4)
hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5)
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(100, (SliceWidth*9)+10)
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12)
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12, (SliceWidth*9)+13, (SliceWidth*9)+14, (SliceWidth*9)+15)
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(2, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4)
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(3, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4, (SliceWidth*9)+5)
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(22, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+10)
hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(100, (SliceWidth*10)+10)
hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12)
hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15)
hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4)
hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5)
hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10)
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4)
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(1, 4)
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5)
hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10)
hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10)
hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10)
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10)
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
// built into cache.Invalidate()
hldr[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).RecalculateCache()
hldr[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).RecalculateCache()
hldr[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).RecalculateCache()
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 10).RecalculateCache()
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).RecalculateCache()
// Connect to each node to compare results.
client := make([]*Client, 3)
@ -96,9 +96,9 @@ func TestClient_MultiNode(t *testing.T) {
client[2] = MustNewClient(s[0].Host())
topN := 4
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f.n", topN)
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN)
result, err := client[0].ExecuteQuery(context.Background(), "d", q, true)
result, err := client[0].ExecuteQuery(context.Background(), "i", q, true)
if err != nil {
t.Fatal(err)
}
@ -106,17 +106,17 @@ func TestClient_MultiNode(t *testing.T) {
// Check the results before every node has the correct max slice value.
pairs := result.(internal.QueryResponse).Results[0].Pairs
for _, pair := range pairs {
if pair.Key == 22 && pair.Count != 5 {
if pair.Key == 22 && pair.Count != 11 {
t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair)
}
}
// Set max slice to correct value.
hldr[0].DB("d").SetRemoteMaxSlice(10)
hldr[1].DB("d").SetRemoteMaxSlice(10)
hldr[2].DB("d").SetRemoteMaxSlice(10)
hldr[0].Index("i").SetRemoteMaxSlice(10)
hldr[1].Index("i").SetRemoteMaxSlice(10)
hldr[2].Index("i").SetRemoteMaxSlice(10)
result, err = client[0].ExecuteQuery(context.Background(), "d", q, true)
result, err = client[0].ExecuteQuery(context.Background(), "i", q, true)
if err != nil {
t.Fatal(err)
}
@ -136,11 +136,11 @@ func TestClient_MultiNode(t *testing.T) {
t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
}
result1, err := client[1].ExecuteQuery(context.Background(), "d", q, true)
result1, err := client[1].ExecuteQuery(context.Background(), "i", q, true)
if err != nil {
t.Fatal(err)
}
result2, err := client[2].ExecuteQuery(context.Background(), "d", q, true)
result2, err := client[2].ExecuteQuery(context.Background(), "i", q, true)
if err != nil {
t.Fatal(err)
}
@ -161,7 +161,7 @@ func TestClient_Import(t *testing.T) {
defer hldr.Close()
// Load bitmap into cache to ensure cache gets updated.
f := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
f.Row(0)
s := NewServer()
@ -173,7 +173,7 @@ func TestClient_Import(t *testing.T) {
// Send import request.
c := MustNewClient(s.Host())
if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
{RowID: 200, ColumnID: 6},
@ -195,7 +195,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
d := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
d := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frameOpts := pilosa.FrameOptions{
InverseEnabled: true,
}
@ -224,7 +224,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
// Send import request.
c := MustNewClient(s.Host())
if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
{RowID: 200, ColumnID: 5},
@ -250,10 +250,10 @@ func TestClient_BackupRestore(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
s := NewServer()
defer s.Close()
@ -266,12 +266,12 @@ func TestClient_BackupRestore(t *testing.T) {
// Backup from frame.
var buf bytes.Buffer
if err := c.BackupTo(context.Background(), &buf, "d", "f", pilosa.ViewStandard); err != nil {
if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewStandard); err != nil {
t.Fatal(err)
}
// Restore to a different frame.
if _, err := hldr.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil {
@ -299,11 +299,11 @@ func TestClient_FragmentBlocks(t *testing.T) {
defer hldr.Close()
// Set two bits on blocks 0 & 3.
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100)
// Set a bit on a different slice.
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
s := NewServer()
defer s.Close()
@ -314,7 +314,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
// Retrieve blocks.
c := MustNewClient(s.Host())
blocks, err := c.FragmentBlocks(context.Background(), "d", "f", pilosa.ViewStandard, 0)
blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0)
if err != nil {
t.Fatal(err)
} else if len(blocks) != 2 {
@ -326,7 +326,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
}
// Verify data matches local blocks.
if a := hldr.Fragment("d", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) {
if a := hldr.Fragment("i", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) {
t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
}
}

View file

@ -146,25 +146,25 @@ func (c *Cluster) NodeByHost(host string) *Node {
}
// Partition returns the partition that a slice belongs to.
func (c *Cluster) Partition(db string, slice uint64) int {
func (c *Cluster) Partition(index string, slice uint64) int {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], slice)
// Hash the bytes and mod by partition count.
h := fnv.New64a()
h.Write([]byte(db))
h.Write([]byte(index))
h.Write(buf[:])
return int(h.Sum64() % uint64(c.PartitionN))
}
// FragmentNodes returns a list of nodes that own a fragment.
func (c *Cluster) FragmentNodes(db string, slice uint64) []*Node {
return c.PartitionNodes(c.Partition(db, slice))
func (c *Cluster) FragmentNodes(index string, slice uint64) []*Node {
return c.PartitionNodes(c.Partition(index, slice))
}
// OwnsFragment returns true if a host owns a fragment.
func (c *Cluster) OwnsFragment(host string, db string, slice uint64) bool {
return Nodes(c.FragmentNodes(db, slice)).ContainsHost(host)
func (c *Cluster) OwnsFragment(host string, index string, slice uint64) bool {
return Nodes(c.FragmentNodes(index, slice)).ContainsHost(host)
}
// PartitionNodes returns a list of nodes that own a partition.

View file

@ -37,11 +37,11 @@ func TestCluster_Owners(t *testing.T) {
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(db string, slice uint64, partitionN int) bool {
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
c := pilosa.NewCluster()
c.PartitionN = partitionN
partitionID := c.Partition(db, slice)
partitionID := c.Partition(index, slice)
if partitionID < 0 || partitionID >= partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
}

View file

@ -29,7 +29,7 @@ Backs up the view from across the cluster into a single file.
}
flags := backupCmd.Flags()
flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Backuper.Database, "database", "d", "", "Pilosa database to backup into.")
flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup into.")
flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup into.")
flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup into.")
flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout")

View file

@ -22,13 +22,13 @@ func TestBackupConfig(t *testing.T) {
args: []string{"backup", "--output-file", "/somefile"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
database = "mydb"
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Backuper.Host, "localhost:12345")
v.Check(cmd.Backuper.Database, "mydb")
v.Check(cmd.Backuper.Index, "myindex")
v.Check(cmd.Backuper.Frame, "f1")
v.Check(cmd.Backuper.Path, "/somefile")
return v.Error()

View file

@ -18,7 +18,7 @@ func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Use: "bench",
Short: "Benchmark operations.",
Long: `
Executes a benchmark for a given operation against the database.
Executes a benchmark for a given operation against the index.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := Bencher.Run(context.Background()); err != nil {
@ -29,7 +29,7 @@ Executes a benchmark for a given operation against the database.
}
flags := benchCmd.Flags()
flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Bencher.Database, "database", "d", "", "Pilosa database to benchmark.")
flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.")
flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.")
flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]")
flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.")

View file

@ -22,13 +22,13 @@ func TestBenchConfig(t *testing.T) {
args: []string{"bench", "--operation", "set-bit"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
database = "mydb"
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Bencher.Host, "localhost:12345")
v.Check(cmd.Bencher.Database, "mydb")
v.Check(cmd.Bencher.Index, "myindex")
v.Check(cmd.Bencher.Frame, "f1")
v.Check(cmd.Bencher.Op, "set-bit")
v.Check(cmd.Bencher.N, 0)

View file

@ -37,7 +37,7 @@ The file does not contain any headers.
flags := exportCmd.Flags()
flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Exporter.Database, "database", "d", "", "Pilosa database to export into.")
flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export into.")
flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export into.")
flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")

View file

@ -22,13 +22,13 @@ func TestExportConfig(t *testing.T) {
args: []string{"export", "--output-file", "/somefile"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
database = "mydb"
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Exporter.Host, "localhost:12345")
v.Check(cmd.Exporter.Database, "mydb")
v.Check(cmd.Exporter.Index, "myindex")
v.Check(cmd.Exporter.Frame, "f1")
v.Check(cmd.Exporter.Path, "/somefile")
return v.Error()

View file

@ -16,7 +16,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
importCmd := &cobra.Command{
Use: "import",
Short: "Bulk load data into pilosa.",
Long: `Bulk imports one or more CSV files to a host's database and frame. The bits
Long: `Bulk imports one or more CSV files to a host's index and frame. The bits
of the CSV file are grouped by slice for the most efficient import.
The format of the CSV file is:
@ -36,7 +36,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
}
flags := importCmd.Flags()
flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Importer.Database, "database", "d", "", "Pilosa database to import into.")
flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.")
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")

View file

@ -22,13 +22,13 @@ func TestImportConfig(t *testing.T) {
args: []string{"import"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
database = "mydb"
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Importer.Host, "localhost:12345")
v.Check(cmd.Importer.Database, "mydb")
v.Check(cmd.Importer.Index, "myindex")
v.Check(cmd.Importer.Frame, "f1")
return v.Error()
},

View file

@ -30,10 +30,10 @@ Restores a view to the cluster from a backup file.
}
flags := restoreCmd.Flags()
flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Restorer.Database, "database", "d", "", "Pilosa database to restore into.")
flags.StringVarP(&Restorer.Index, "index", "i", "", "Pilosa index to restore into.")
flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.")
flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.")
flags.StringVarP(&Restorer.Path, "input-file", "i", "", "File to restore from.")
flags.StringVarP(&Restorer.Path, "input-file", "d", "", "File to restore data from.")
return restoreCmd
}

View file

@ -22,13 +22,13 @@ func TestRestoreConfig(t *testing.T) {
args: []string{"restore", "--input-file", "/somefile"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
database = "mydb"
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Restorer.Host, "localhost:12345")
v.Check(cmd.Restorer.Database, "mydb")
v.Check(cmd.Restorer.Index, "myindex")
v.Check(cmd.Restorer.Frame, "f1")
v.Check(cmd.Restorer.Path, "/somefile")
return v.Error()

View file

@ -14,10 +14,10 @@ type BackupCommand struct {
// Destination host and port.
Host string
// Name of the database, frame, view to backup.
Database string
Frame string
View string
// Name of the index, frame, view to backup.
Index string
Frame string
View string
// Output file to write to.
Path string
@ -54,7 +54,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) error {
defer f.Close()
// Begin streaming backup.
if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil {
if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
return err
}

View file

@ -11,14 +11,14 @@ import (
"github.com/pilosa/pilosa"
)
// BenchCommand represents a command for benchmarking database operations.
// BenchCommand represents a command for benchmarking index operations.
type BenchCommand struct {
// Destination host and port.
Host string
// Name of the database & frame to execute against.
Database string
Frame string
// Name of the index & frame to execute against.
Index string
Frame string
// Type of operation and number to execute.
Op string
@ -57,8 +57,8 @@ func (cmd *BenchCommand) Run(ctx context.Context) error {
func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error {
if cmd.N == 0 {
return errors.New("operation count required")
} else if cmd.Database == "" {
return pilosa.ErrDatabaseRequired
} else if cmd.Index == "" {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
}
@ -75,7 +75,7 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e
q := fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID)
if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil {
if _, err := client.ExecuteQuery(ctx, cmd.Index, q, true); err != nil {
return err
}
}

View file

@ -14,9 +14,9 @@ type ExportCommand struct {
// Remote host and port.
Host string
// Name of the database & frame to export from.
Database string
Frame string
// Name of the index & frame to export from.
Index string
Frame string
// Filename to export to.
Path string
@ -37,8 +37,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// Validate arguments.
if cmd.Database == "" {
return pilosa.ErrDatabaseRequired
if cmd.Index == "" {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
}
@ -63,15 +63,15 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
}
// Determine slice count.
maxSlices, err := client.MaxSliceByDatabase(ctx)
maxSlices, err := client.MaxSliceByIndex(ctx)
if err != nil {
return err
}
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Database]; slice++ {
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Database, cmd.Frame, slice, w); err != nil {
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil {
return err
}
}

View file

@ -19,9 +19,9 @@ type ImportCommand struct {
// Destination host and port.
Host string `json:"host"`
// Name of the database & frame to import into.
Database string `json:"db"`
Frame string `json:"frame"`
// Name of the index & frame to import into.
Index string `json:"index"`
Frame string `json:"frame"`
// Filenames to import from.
Paths []string `json:"paths"`
@ -54,9 +54,9 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// Validate arguments.
// Database and frame are validated early before the files are parsed.
if cmd.Database == "" {
return pilosa.ErrDatabaseRequired
// Index and frame are validated early before the files are parsed.
if cmd.Index == "" {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
} else if len(cmd.Paths) == 0 {
@ -176,7 +176,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
// Parse path into bits.
for slice, bits := range bitsBySlice {
logger.Printf("importing slice: %d, n=%d", slice, len(bits))
if err := cmd.Client.Import(ctx, cmd.Database, cmd.Frame, slice, bits); err != nil {
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil {
return err
}
}

View file

@ -14,10 +14,10 @@ type RestoreCommand struct {
// Destination host and port.
Host string
// Name of the database & frame to backup.
Database string
Frame string
View string
// Name of the index & frame to backup.
Index string
Frame string
View string
// Import file to read from.
Path string
@ -54,7 +54,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
defer f.Close()
// Restore backup file to the cluster.
if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil {
if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
return err
}

565
db.go
View file

@ -1,565 +0,0 @@
package pilosa
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// Default database settings.
const (
DefaultColumnLabel = "columnID"
)
// DB represents a container for frames.
type DB struct {
mu sync.Mutex
path string
name string
// Default time quantum for all frames in database.
// This can be overridden by individual frames.
timeQuantum TimeQuantum
// Label used for referring to columns in database.
columnLabel string
// Frames by name.
frames map[string]*Frame
// Max Slice on any node in the cluster, according to this node
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
// Column attribute storage and cache
columnAttrStore *AttrStore
broadcaster Broadcaster
stats StatsClient
LogOutput io.Writer
}
// NewDB returns a new instance of DB.
func NewDB(path, name string) (*DB, error) {
err := ValidateName(name)
if err != nil {
return nil, err
}
return &DB{
path: path,
name: name,
frames: make(map[string]*Frame),
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
columnAttrStore: NewAttrStore(filepath.Join(path, ".data")),
columnLabel: DefaultColumnLabel,
stats: NopStatsClient,
LogOutput: ioutil.Discard,
}, nil
}
// Name returns name of the database.
func (db *DB) Name() string { return db.name }
// Path returns the path the database was initialized with.
func (db *DB) Path() string { return db.path }
// ColumnAttrStore returns the storage for column attributes.
func (db *DB) ColumnAttrStore() *AttrStore { return db.columnAttrStore }
// SetColumnLabel sets the column label. Persists to meta file on update.
func (db *DB) SetColumnLabel(v string) error {
db.mu.Lock()
defer db.mu.Unlock()
// Ignore if no change occurred.
if v == "" || db.columnLabel == v {
return nil
}
// Make sure columnLabel is valid name
err := ValidateName(v)
if err != nil {
return err
}
// Persist meta data to disk on change.
db.columnLabel = v
if err := db.saveMeta(); err != nil {
return err
}
return nil
}
// ColumnLabel returns the column label.
func (db *DB) ColumnLabel() string {
db.mu.Lock()
v := db.columnLabel
db.mu.Unlock()
return v
}
// Open opens and initializes the database.
func (db *DB) Open() error {
// Ensure the path exists.
if err := os.MkdirAll(db.path, 0777); err != nil {
return err
}
// Read meta file.
if err := db.loadMeta(); err != nil {
return err
}
if err := db.openFrames(); err != nil {
return err
}
if err := db.columnAttrStore.Open(); err != nil {
return err
}
return nil
}
// openFrames opens and initializes the frames inside the database.
func (db *DB) openFrames() error {
f, err := os.Open(db.path)
if err != nil {
return err
}
defer f.Close()
fis, err := f.Readdir(0)
if err != nil {
return err
}
for _, fi := range fis {
if !fi.IsDir() {
continue
}
fr, err := db.newFrame(db.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err != nil {
return ErrName
}
if err := fr.Open(); err != nil {
return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err)
}
db.frames[fr.Name()] = fr
db.stats.Count("frameN", 1)
}
return nil
}
// loadMeta reads meta data for the database, if any.
func (db *DB) loadMeta() error {
var pb internal.DBMeta
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(db.path, ".meta"))
if os.IsNotExist(err) {
db.timeQuantum = ""
db.columnLabel = DefaultColumnLabel
return nil
} else if err != nil {
return err
} else {
if err := proto.Unmarshal(buf, &pb); err != nil {
return err
}
}
// Copy metadata fields.
db.timeQuantum = TimeQuantum(pb.TimeQuantum)
db.columnLabel = pb.ColumnLabel
return nil
}
// saveMeta writes meta data for the database.
func (db *DB) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.DBMeta{
TimeQuantum: string(db.timeQuantum),
ColumnLabel: db.columnLabel,
})
if err != nil {
return err
}
// Write to meta file.
if err := ioutil.WriteFile(filepath.Join(db.path, ".meta"), buf, 0666); err != nil {
return err
}
return nil
}
// Close closes the database and its frames.
func (db *DB) Close() error {
db.mu.Lock()
defer db.mu.Unlock()
// Close the attribute store.
if db.columnAttrStore != nil {
db.columnAttrStore.Close()
}
// Close all frames.
for _, f := range db.frames {
f.Close()
}
db.frames = make(map[string]*Frame)
return nil
}
// MaxSlice returns the max slice in the database according to this node.
func (db *DB) MaxSlice() uint64 {
if db == nil {
return 0
}
db.mu.Lock()
defer db.mu.Unlock()
max := db.remoteMaxSlice
for _, f := range db.frames {
if slice := f.MaxSlice(); slice > max {
max = slice
}
}
return max
}
func (db *DB) SetRemoteMaxSlice(newmax uint64) {
db.mu.Lock()
defer db.mu.Unlock()
db.remoteMaxSlice = newmax
}
// MaxInverseSlice returns the max inverse slice in the database according to this node.
func (db *DB) MaxInverseSlice() uint64 {
if db == nil {
return 0
}
db.mu.Lock()
defer db.mu.Unlock()
max := db.remoteMaxInverseSlice
for _, f := range db.frames {
if slice := f.MaxInverseSlice(); slice > max {
max = slice
}
}
return max
}
func (db *DB) SetRemoteMaxInverseSlice(v uint64) {
db.mu.Lock()
defer db.mu.Unlock()
db.remoteMaxInverseSlice = v
}
// TimeQuantum returns the default time quantum for the database.
func (db *DB) TimeQuantum() TimeQuantum {
db.mu.Lock()
defer db.mu.Unlock()
return db.timeQuantum
}
// SetTimeQuantum sets the default time quantum for the database.
func (db *DB) SetTimeQuantum(q TimeQuantum) error {
db.mu.Lock()
defer db.mu.Unlock()
// Validate input.
if !q.Valid() {
return ErrInvalidTimeQuantum
}
// Update value on database.
db.timeQuantum = q
// Perist meta data to disk.
if err := db.saveMeta(); err != nil {
return err
}
return nil
}
// FramePath returns the path to a frame in the database.
func (db *DB) FramePath(name string) string { return filepath.Join(db.path, name) }
// Frame returns a frame in the database by name.
func (db *DB) Frame(name string) *Frame {
db.mu.Lock()
defer db.mu.Unlock()
return db.frame(name)
}
func (db *DB) frame(name string) *Frame { return db.frames[name] }
// Frames returns a list of all frames in the database.
func (db *DB) Frames() []*Frame {
db.mu.Lock()
defer db.mu.Unlock()
a := make([]*Frame, 0, len(db.frames))
for _, f := range db.frames {
a = append(a, f)
}
sort.Sort(frameSlice(a))
return a
}
// CreateFrame creates a frame.
func (db *DB) CreateFrame(name string, opt FrameOptions) (*Frame, error) {
db.mu.Lock()
defer db.mu.Unlock()
// Ensure frame doesn't already exist.
if db.frames[name] != nil {
return nil, ErrFrameExists
}
return db.createFrame(name, opt)
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (db *DB) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) {
db.mu.Lock()
defer db.mu.Unlock()
// Find frame in cache first.
if f := db.frames[name]; f != nil {
return f, nil
}
return db.createFrame(name, opt)
}
func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) {
if name == "" {
return nil, errors.New("frame name required")
} else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) {
return nil, ErrInvalidCacheType
}
// Initialize frame.
f, err := db.newFrame(db.FramePath(name), name)
if err != nil {
return nil, err
}
// Open frame.
if err := f.Open(); err != nil {
return nil, err
}
// Default the time quantum to what is set on the DB.
if err := f.SetTimeQuantum(db.timeQuantum); err != nil {
f.Close()
return nil, err
}
// Set cache type.
if opt.CacheType == "" {
opt.CacheType = DefaultCacheType
}
f.cacheType = opt.CacheType
// Set options.
if opt.RowLabel != "" {
f.rowLabel = opt.RowLabel
}
if opt.CacheSize != 0 {
f.cacheSize = opt.CacheSize
}
f.inverseEnabled = opt.InverseEnabled
if err := f.saveMeta(); err != nil {
f.Close()
return nil, err
}
// Add to database's frame lookup.
db.frames[name] = f
db.stats.Count("frameN", 1)
return f, nil
}
func (db *DB) newFrame(path, name string) (*Frame, error) {
f, err := NewFrame(path, db.name, name)
if err != nil {
return nil, err
}
f.LogOutput = db.LogOutput
f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name))
f.broadcaster = db.broadcaster
return f, nil
}
// DeleteFrame removes a frame from the database.
func (db *DB) DeleteFrame(name string) error {
db.mu.Lock()
defer db.mu.Unlock()
// Ignore if frame doesn't exist.
f := db.frame(name)
if f == nil {
return nil
}
// Close frame.
if err := f.Close(); err != nil {
return err
}
// Delete frame directory.
if err := os.RemoveAll(db.FramePath(name)); err != nil {
return err
}
// Remove reference.
delete(db.frames, name)
db.stats.Count("frameN", -1)
return nil
}
type dbSlice []*DB
func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p dbSlice) Len() int { return len(p) }
func (p dbSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// DBInfo represents schema information for a database.
type DBInfo struct {
Name string `json:"name"`
Frames []*FrameInfo `json:"frames"`
}
type dbInfoSlice []*DBInfo
func (p dbInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p dbInfoSlice) Len() int { return len(p) }
func (p dbInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// MergeSchemas combines databases and frames from a and b into one schema.
func MergeSchemas(a, b []*DBInfo) []*DBInfo {
// Generate a map from both schemas.
m := make(map[string]map[string]map[string]struct{})
for _, dbs := range [][]*DBInfo{a, b} {
for _, db := range dbs {
if m[db.Name] == nil {
m[db.Name] = make(map[string]map[string]struct{})
}
for _, frame := range db.Frames {
if m[db.Name][frame.Name] == nil {
m[db.Name][frame.Name] = make(map[string]struct{})
}
for _, view := range frame.Views {
m[db.Name][frame.Name][view.Name] = struct{}{}
}
}
}
}
// Generate new schema from map.
dbs := make([]*DBInfo, 0, len(m))
for db, frames := range m {
di := &DBInfo{Name: db}
for frame, views := range frames {
fi := &FrameInfo{Name: frame}
for view := range views {
fi.Views = append(fi.Views, &ViewInfo{Name: view})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Frames = append(di.Frames, fi)
}
sort.Sort(frameInfoSlice(di.Frames))
dbs = append(dbs, di)
}
sort.Sort(dbInfoSlice(dbs))
return dbs
}
// encodeDBs converts a into its internal representation.
func encodeDBs(a []*DB) []*internal.DB {
other := make([]*internal.DB, len(a))
for i := range a {
other[i] = encodeDB(a[i])
}
return other
}
// encodeDB converts d into its internal representation.
func encodeDB(d *DB) *internal.DB {
return &internal.DB{
Name: d.name,
Meta: &internal.DBMeta{
ColumnLabel: d.columnLabel,
TimeQuantum: string(d.timeQuantum),
},
MaxSlice: d.remoteMaxSlice,
Frames: encodeFrames(d.Frames()),
}
}
// DBOptions represents options to set when initializing a db.
type DBOptions struct {
ColumnLabel string `json:"columnLabel,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
}
// Encode converts o into its internal representation.
func (o *DBOptions) Encode() *internal.DBMeta {
return &internal.DBMeta{
ColumnLabel: o.ColumnLabel,
TimeQuantum: string(o.TimeQuantum),
}
}
// hasTime returns true if a contains a non-nil time.
func hasTime(a []*time.Time) bool {
for _, t := range a {
if t != nil {
return true
}
}
return false
}
type importKey struct {
View string
Slice uint64
}
type importData struct {
RowIDs []uint64
ColumnIDs []uint64
}

View file

@ -1,179 +0,0 @@
package pilosa_test
import (
"io/ioutil"
"os"
"testing"
"github.com/pilosa/pilosa"
)
// Ensure database can open and retrieve a frame.
func TestDB_CreateFrameIfNotExists(t *testing.T) {
db := MustOpenDB()
defer db.Close()
// Create frame.
f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if f == nil {
t.Fatal("expected frame")
}
// Retrieve existing frame.
other, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if f.Frame != other.Frame {
t.Fatal("frame mismatch")
}
if f.Frame != db.Frame("f") {
t.Fatal("frame mismatch")
}
}
// Ensure database defaults the time quantum on new frames.
func TestDB_CreateFrame_TimeQuantum(t *testing.T) {
db := MustOpenDB()
defer db.Close()
// Set database time quantum.
if err := db.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
t.Fatal(err)
}
// Create frame.
f, err := db.CreateFrame("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") {
t.Fatalf("unexpected frame time quantum: %s", q)
}
}
// Ensure database can delete a frame.
func TestDB_DeleteFrame(t *testing.T) {
db := MustOpenDB()
defer db.Close()
// Create frame.
if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Delete frame & verify it's gone.
if err := db.DeleteFrame("f"); err != nil {
t.Fatal(err)
} else if db.Frame("f") != nil {
t.Fatal("expected nil frame")
}
// Delete again to make sure it doesn't error.
if err := db.DeleteFrame("f"); err != nil {
t.Fatal(err)
}
}
// Ensure database can set the default time quantum.
func TestDB_SetTimeQuantum(t *testing.T) {
db := MustOpenDB()
defer db.Close()
// Set & retrieve time quantum.
if err := db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload database and verify that it is persisted.
if err := db.Reopen(); err != nil {
t.Fatal(err)
} else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// DB represents a test wrapper for pilosa.DB.
type DB struct {
*pilosa.DB
}
// NewDB returns a new instance of DB d.
func NewDB() *DB {
path, err := ioutil.TempDir("", "pilosa-db-")
if err != nil {
panic(err)
}
db, err := pilosa.NewDB(path, "d")
if err != nil {
panic(err)
}
return &DB{DB: db}
}
// MustOpenDB returns a new, opened database at a temporary path. Panic on error.
func MustOpenDB() *DB {
db := NewDB()
if err := db.Open(); err != nil {
panic(err)
}
return db
}
// Close closes the database and removes the underlying data.
func (db *DB) Close() error {
defer os.RemoveAll(db.Path())
return db.DB.Close()
}
// Reopen closes the database and reopens it.
func (db *DB) Reopen() error {
var err error
if err := db.DB.Close(); err != nil {
return err
}
path, name := db.Path(), db.Name()
db.DB, err = pilosa.NewDB(path, name)
if err != nil {
return err
}
if err := db.Open(); err != nil {
return err
}
return nil
}
// CreateFrame creates a frame with the given options.
func (db *DB) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := db.DB.CreateFrame(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (db *DB) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := db.DB.CreateFrameIfNotExists(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
}
// Ensure database can delete a frame.
func TestDB_InvalidName(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-db-")
if err != nil {
panic(err)
}
db, err := pilosa.NewDB(path, "ABC")
if db != nil {
t.Fatalf("unexpected db name %s", db)
}
}

View file

@ -45,10 +45,10 @@ func NewExecutor() *Executor {
}
// Execute executes a PQL query.
func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) {
// Verify that a database is set.
if db == "" {
return nil, ErrDatabaseRequired
func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) {
// Verify that an index is set.
if index == "" {
return nil, ErrIndexRequired
}
// Default options.
@ -60,7 +60,7 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices
if len(slices) == 0 {
if needsSlices(q.Calls) {
// Round up the number of slices.
maxSlice := e.Holder.DB(db).MaxSlice()
maxSlice := e.Holder.Index(index).MaxSlice()
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
@ -72,13 +72,13 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices
// Optimize handling for bulk attribute insertion.
if hasOnlySetRowAttrs(q.Calls) {
return e.executeBulkSetRowAttrs(ctx, db, q.Calls, opt)
return e.executeBulkSetRowAttrs(ctx, index, q.Calls, opt)
}
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
v, err := e.executeCall(ctx, db, call, slices, opt)
v, err := e.executeCall(ctx, index, call, slices, opt)
if err != nil {
return nil, err
}
@ -88,7 +88,7 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices
}
// executeCall executes a call.
func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
if err := e.validateCallArgs(c); err != nil {
return nil, err
@ -97,19 +97,19 @@ func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slic
// Special handling for mutation and top-n calls.
switch c.Name {
case "ClearBit":
return e.executeClearBit(ctx, db, c, opt)
return e.executeClearBit(ctx, index, c, opt)
case "Count":
return e.executeCount(ctx, db, c, slices, opt)
return e.executeCount(ctx, index, c, slices, opt)
case "SetBit":
return e.executeSetBit(ctx, db, c, opt)
return e.executeSetBit(ctx, index, c, opt)
case "SetRowAttrs":
return nil, e.executeSetRowAttrs(ctx, db, c, opt)
return nil, e.executeSetRowAttrs(ctx, index, c, opt)
case "SetColumnAttrs":
return nil, e.executeSetColumnAttrs(ctx, db, c, opt)
return nil, e.executeSetColumnAttrs(ctx, index, c, opt)
case "TopN":
return e.executeTopN(ctx, db, c, slices, opt)
return e.executeTopN(ctx, index, c, slices, opt)
default:
return e.executeBitmapCall(ctx, db, c, slices, opt)
return e.executeBitmapCall(ctx, index, c, slices, opt)
}
}
@ -133,10 +133,10 @@ func (e *Executor) validateCallArgs(c *pql.Call) error {
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) {
func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) {
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeBitmapCallSlice(ctx, db, c, slice)
return e.executeBitmapCallSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
@ -149,7 +149,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
return other
}
other, err := e.mapReduce(ctx, db, slices, c, opt, mapFn, reduceFn)
other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
}
@ -160,7 +160,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
bm, _ := other.(*Bitmap)
if c.Name == "Bitmap" {
d := e.Holder.DB(db)
d := e.Holder.Index(index)
if d != nil {
columnLabel := d.ColumnLabel()
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
@ -193,18 +193,18 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
}
// executeBitmapCallSlice executes a bitmap call for a single slice.
func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
switch c.Name {
case "Bitmap":
return e.executeBitmapSlice(ctx, db, c, slice)
return e.executeBitmapSlice(ctx, index, c, slice)
case "Difference":
return e.executeDifferenceSlice(ctx, db, c, slice)
return e.executeDifferenceSlice(ctx, index, c, slice)
case "Intersect":
return e.executeIntersectSlice(ctx, db, c, slice)
return e.executeIntersectSlice(ctx, index, c, slice)
case "Range":
return e.executeRangeSlice(ctx, db, c, slice)
return e.executeRangeSlice(ctx, index, c, slice)
case "Union":
return e.executeUnionSlice(ctx, db, c, slice)
return e.executeUnionSlice(ctx, index, c, slice)
default:
return nil, fmt.Errorf("unknown call: %s", c.Name)
}
@ -213,7 +213,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql
// executeTopN executes a TopN() call.
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
rowIDs, _, err := c.UintSliceArg("ids")
if err != nil {
return nil, fmt.Errorf("executeTopN: %v", err)
@ -224,7 +224,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic
}
// Execute original query.
pairs, err := e.executeTopNSlices(ctx, db, c, slices, opt)
pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt)
if err != nil {
return nil, err
}
@ -241,7 +241,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic
sort.Sort(uint64Slice(ids))
other.Args["ids"] = ids
trimmedList, err := e.executeTopNSlices(ctx, db, other, slices, opt)
trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt)
if err != nil {
return nil, err
}
@ -252,10 +252,10 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic
return trimmedList, nil
}
func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeTopNSlice(ctx, db, c, slice)
return e.executeTopNSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
@ -264,7 +264,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call
return Pairs(other).Add(v.([]Pair))
}
other, err := e.mapReduce(ctx, db, slices, c, opt, mapFn, reduceFn)
other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return nil, err
}
@ -277,7 +277,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call
}
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, slice uint64) ([]Pair, error) {
func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
frame, _ := c.Args["frame"].(string)
n, _, err := c.UintArg("n")
if err != nil {
@ -301,7 +301,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
// Retrieve bitmap used to intersect.
var src *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice)
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return nil, err
}
@ -315,7 +315,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
frame = DefaultFrame
}
f := e.Holder.Fragment(db, frame, ViewStandard, slice)
f := e.Holder.Fragment(index, frame, ViewStandard, slice)
if f == nil {
return nil, nil
}
@ -339,13 +339,13 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
}
// executeDifferenceSlice executes a difference() call for a local slice.
func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
var other *Bitmap
if len(c.Children) == 0 {
return nil, fmt.Errorf("empty Difference query is currently not supported")
}
for i, input := range c.Children {
bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
bm, err := e.executeBitmapCallSlice(ctx, index, input, slice)
if err != nil {
return nil, err
}
@ -360,11 +360,11 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql
return other, nil
}
func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
// Fetch column label from database.
d := e.Holder.DB(db)
func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
// Fetch column label from index.
d := e.Holder.Index(index)
if d == nil {
return nil, ErrDatabaseNotFound
return nil, ErrIndexNotFound
}
columnLabel := d.ColumnLabel()
@ -373,7 +373,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal
if frame == "" {
frame = DefaultFrame
}
f := e.Holder.Frame(db, frame)
f := e.Holder.Frame(index, frame)
if f == nil {
return nil, ErrFrameNotFound
}
@ -400,7 +400,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal
}
}
frag := e.Holder.Fragment(db, frame, view, slice)
frag := e.Holder.Fragment(index, frame, view, slice)
if frag == nil {
return NewBitmap(), nil
}
@ -408,13 +408,13 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal
}
// executeIntersectSlice executes a intersect() call for a local slice.
func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
var other *Bitmap
if len(c.Children) == 0 {
return nil, fmt.Errorf("empty Intersect query is currently not supported")
}
for i, input := range c.Children {
bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
bm, err := e.executeBitmapCallSlice(ctx, index, input, slice)
if err != nil {
return nil, err
}
@ -430,7 +430,7 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.
}
// executeRangeSlice executes a range() call for a local slice.
func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
// Parse frame, use default if unset.
frame, _ := c.Args["frame"].(string)
if frame == "" {
@ -438,7 +438,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
}
// Retrieve base frame.
f := e.Holder.Frame(db, frame)
f := e.Holder.Frame(index, frame)
if f == nil {
return nil, ErrFrameNotFound
}
@ -479,7 +479,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
// Union bitmaps across all time-based subframes.
bm := &Bitmap{}
for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(db, frame, view, slice)
f := e.Holder.Fragment(index, frame, view, slice)
if f == nil {
continue
}
@ -489,10 +489,10 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
}
// executeUnionSlice executes a union() call for a local slice.
func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
other := NewBitmap()
for i, input := range c.Children {
bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
bm, err := e.executeBitmapCallSlice(ctx, index, input, slice)
if err != nil {
return nil, err
}
@ -508,7 +508,7 @@ func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Call
}
// executeCount executes a count() call.
func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) {
func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) {
if len(c.Children) == 0 {
return 0, errors.New("Count() requires an input bitmap")
} else if len(c.Children) > 1 {
@ -517,7 +517,7 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, sli
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice)
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return 0, err
}
@ -530,7 +530,7 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, sli
return other + v.(uint64)
}
result, err := e.mapReduce(ctx, db, slices, c, opt, mapFn, reduceFn)
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
@ -540,7 +540,7 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, sli
}
// executeClearBit executes a ClearBit() call.
func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) {
func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
view, _ := c.Args["view"].(string)
frame, ok := c.Args["frame"].(string)
if !ok {
@ -548,9 +548,9 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
}
// Retrieve frame.
d := e.Holder.DB(db)
d := e.Holder.Index(index)
if d == nil {
return false, ErrDatabaseNotFound
return false, ErrIndexNotFound
}
f := d.Frame(frame)
if f == nil {
@ -579,19 +579,19 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
// Clear bits for each view.
switch view {
case ViewStandard:
return e.executeClearBitView(ctx, db, c, f, view, colID, rowID, opt)
return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt)
case ViewInverse:
return e.executeClearBitView(ctx, db, c, f, view, rowID, colID, opt)
return e.executeClearBitView(ctx, index, c, f, view, rowID, colID, opt)
case "":
var ret bool
if changed, err := e.executeClearBitView(ctx, db, c, f, ViewStandard, colID, rowID, opt); err != nil {
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
if f.InverseEnabled() {
if changed, err := e.executeClearBitView(ctx, db, c, f, ViewInverse, rowID, colID, opt); err != nil {
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewInverse, rowID, colID, opt); err != nil {
return ret, err
} else if changed {
ret = true
@ -604,10 +604,10 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
}
// executeClearBitView executes a ClearBit() call for a single view.
func (e *Executor) executeClearBitView(ctx context.Context, db string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(db, slice) {
for _, node := range e.Cluster.FragmentNodes(index, slice) {
// Update locally if host matches.
if node.Host == e.Host {
val, err := f.ClearBit(view, rowID, colID, nil)
@ -624,7 +624,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, db string, c *pql.Ca
}
// Forward call to remote node otherwise.
if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res[0].(bool)
@ -634,7 +634,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, db string, c *pql.Ca
}
// executeSetBit executes a SetBit() call.
func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) {
func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
view, _ := c.Args["view"].(string)
frame, ok := c.Args["frame"].(string)
if !ok {
@ -642,9 +642,9 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
}
// Retrieve frame.
d := e.Holder.DB(db)
d := e.Holder.Index(index)
if d == nil {
return false, ErrDatabaseNotFound
return false, ErrIndexNotFound
}
f := d.Frame(frame)
if f == nil {
@ -683,19 +683,19 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
// Set bits for each view.
switch view {
case ViewStandard:
return e.executeSetBitView(ctx, db, c, f, view, colID, rowID, timestamp, opt)
return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt)
case ViewInverse:
return e.executeSetBitView(ctx, db, c, f, view, rowID, colID, timestamp, opt)
return e.executeSetBitView(ctx, index, c, f, view, rowID, colID, timestamp, opt)
case "":
var ret bool
if changed, err := e.executeSetBitView(ctx, db, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil {
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
if f.InverseEnabled() {
if changed, err := e.executeSetBitView(ctx, db, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil {
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil {
return ret, err
} else if changed {
ret = true
@ -708,11 +708,11 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
}
// executeSetBitView executes a SetBit() call for a specific view.
func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) {
func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) {
slice := colID / SliceWidth
ret := false
for _, node := range e.Cluster.FragmentNodes(db, slice) {
for _, node := range e.Cluster.FragmentNodes(index, slice) {
// Update locally if host matches.
if node.Host == e.Host {
val, err := f.SetBit(view, rowID, colID, timestamp)
@ -730,7 +730,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call
}
// Forward call to remote node otherwise.
if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
return false, err
} else {
ret = res[0].(bool)
@ -740,14 +740,14 @@ func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call
}
// executeSetRowAttrs executes a SetRowAttrs() call.
func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error {
frameName, ok := c.Args["frame"].(string)
if !ok {
return errors.New("SetRowAttrs() frame required")
}
// Retrieve frame.
frame := e.Holder.Frame(db, frameName)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return ErrFrameNotFound
}
@ -781,7 +781,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Cal
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
_, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
resp <- err
}(node)
}
@ -797,7 +797,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Cal
}
// executeBulkSetRowAttrs executes a set of SetRowAttrs() calls.
func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) {
func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) {
// Collect attributes by frame/id.
m := make(map[string]map[uint64]map[string]interface{})
for _, c := range calls {
@ -807,7 +807,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls
}
// Retrieve frame.
f := e.Holder.Frame(db, frame)
f := e.Holder.Frame(index, frame)
if f == nil {
return nil, ErrFrameNotFound
}
@ -846,7 +846,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls
// Bulk insert attributes by frame.
for name, frameMap := range m {
// Retrieve frame.
frame := e.Holder.Frame(db, name)
frame := e.Holder.Frame(index, name)
if frame == nil {
return nil, ErrFrameNotFound
}
@ -867,7 +867,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.exec(ctx, node, db, &pql.Query{Calls: calls}, nil, opt)
_, err := e.exec(ctx, node, index, &pql.Query{Calls: calls}, nil, opt)
resp <- err
}(node)
}
@ -884,11 +884,11 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls
}
// executeSetColumnAttrs executes a SetColumnAttrs() call.
func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
// Retrieve database.
d := e.Holder.DB(db)
func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error {
// Retrieve index.
d := e.Holder.Index(index)
if d == nil {
return ErrDatabaseNotFound
return ErrIndexNotFound
}
var colName string
@ -925,7 +925,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql.
resp := make(chan error, len(nodes))
for _, node := range nodes {
go func(node *Node) {
_, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
_, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
resp <- err
}(node)
}
@ -941,7 +941,7 @@ func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql.
}
// exec executes a PQL query remotely for a set of slices on a node.
func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) {
func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) {
// Encode request object.
pbreq := &internal.QueryRequest{
Query: q.String(),
@ -957,7 +957,7 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query
req, err := http.NewRequest("POST", (&url.URL{
Scheme: "http",
Host: node.Host,
Path: fmt.Sprintf("/db/%s/query", db),
Path: fmt.Sprintf("/index/%s/query", index),
}).String(), bytes.NewReader(buf))
if err != nil {
return nil, err
@ -1027,12 +1027,12 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query
// slicesByNode returns a mapping of nodes to slices.
// Returns errSliceUnavailable if a slice cannot be allocated to a node.
func (e *Executor) slicesByNode(nodes []*Node, db string, slices []uint64) (map[*Node][]uint64, error) {
func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (map[*Node][]uint64, error) {
m := make(map[*Node][]uint64)
loop:
for _, slice := range slices {
for _, node := range e.Cluster.FragmentNodes(db, slice) {
for _, node := range e.Cluster.FragmentNodes(index, slice) {
if Nodes(nodes).Contains(node) {
m[node] = append(m[node], slice)
continue loop
@ -1047,7 +1047,7 @@ loop:
//
// If a mapping of slices to a node fails then the slices are resplit across
// secondary nodes and retried. This continues to occur until all nodes are exhausted.
func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
ch := make(chan mapResponse, 0)
// Wrap context with a cancel to kill goroutines on exit.
@ -1066,7 +1066,7 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c
}
// Start mapping across all primary owners.
if err := e.mapper(ctx, ch, nodes, db, slices, c, opt, mapFn, reduceFn); err != nil {
if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil {
return nil, err
}
@ -1085,7 +1085,7 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c
nodes = Nodes(nodes).Filter(resp.node)
// Begin mapper against secondary nodes.
if err := e.mapper(ctx, ch, nodes, db, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable {
if err := e.mapper(ctx, ch, nodes, index, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable {
return nil, resp.err
} else if err != nil {
return nil, err
@ -1105,9 +1105,9 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c
}
}
func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error {
func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error {
// Group slices together by nodes.
m, err := e.slicesByNode(nodes, db, slices)
m, err := e.slicesByNode(nodes, index, slices)
if err != nil {
return err
}
@ -1122,7 +1122,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.exec(ctx, n, db, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
if len(results) > 0 {
resp.result = results[0]
}

View file

@ -17,8 +17,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Run("Row", func(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
f, err := db.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
if err != nil {
t.Fatal(err)
}
@ -26,7 +26,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
e := NewExecutor(hldr.Holder, NewCluster(1))
// Set bits.
if _, err := e.Execute(context.Background(), "d", MustParse(``+
if _, err := e.Execute(context.Background(), "i", MustParse(``+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1),
@ -37,7 +37,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -49,26 +49,26 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Run("Column", func(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
if _, err := db.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, NewCluster(1))
// Set bits.
if _, err := e.Execute(context.Background(), "d", MustParse(``+
if _, err := e.Execute(context.Background(), "i", MustParse(``+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1),
), nil, nil); err != nil {
t.Fatal(err)
}
if err := db.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "d", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -82,14 +82,14 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
func TestExecutor_Execute_Difference(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -100,10 +100,10 @@ func TestExecutor_Execute_Difference(t *testing.T) {
func TestExecutor_Execute_Empty_Difference(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Difference()`), nil, nil); err == nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Difference()`), nil, nil); err == nil {
t.Fatalf("Empty Difference query should give error, but got %v", res)
}
}
@ -112,16 +112,16 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) {
func TestExecutor_Execute_Intersect(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -134,7 +134,7 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
defer hldr.Close()
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect()`), nil, nil); err == nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect()`), nil, nil); err == nil {
t.Fatalf("Empty Intersect query should give error, but got %v", res)
}
}
@ -143,15 +143,15 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
func TestExecutor_Execute_Union(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -162,10 +162,10 @@ func TestExecutor_Execute_Union(t *testing.T) {
func TestExecutor_Execute_Empty_Union(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Union()`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Union()`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -176,12 +176,12 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
func TestExecutor_Execute_Count(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(3) {
t.Fatalf("unexpected n: %d", res[0])
@ -194,12 +194,12 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
defer hldr.Close()
e := NewExecutor(hldr.Holder, NewCluster(1))
f := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if n := f.Row(11).Count(); n != 0 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if !res[0].(bool) {
@ -210,7 +210,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if n := f.Row(11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if res[0].(bool) {
@ -225,30 +225,30 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
defer hldr.Close()
// Create frames.
db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := db.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil {
} else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Set two fields on f/10.
// Also set fields on other bitmaps and frames to test isolation.
e := NewExecutor(hldr.Holder, NewCluster(1))
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
}
f := hldr.Frame("d", "f")
f := hldr.Frame("i", "f")
if m, err := f.RowAttrStore().Attrs(10); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
@ -262,19 +262,19 @@ func TestExecutor_Execute_TopN(t *testing.T) {
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{ID: 0, Count: 5},
@ -288,16 +288,16 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 4},
@ -311,27 +311,27 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
@ -346,23 +346,23 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
// Create an intersecting row.
hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 20, Count: 3},
@ -378,15 +378,15 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
//
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := hldr.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -401,15 +401,15 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
//
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := hldr.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(Bitmap(id=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(id=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -424,11 +424,11 @@ func TestExecutor_Execute_Range(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
// Create database.
db := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
// Create index.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
// Create frame.
f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
@ -448,7 +448,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different row
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -465,14 +465,12 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Bitmap(frame="f", id=10)` {
t.Fatalf("unexpected query: %s", query.String())
// NOTE: while the following is technically incorrect (it should be {0, 2}) because the calling node doesn't know about slice 2 yet,
// we are ok with this and assuming that the calling node will become aware of slice 2 via inter-node messaging
} else if !reflect.DeepEqual(slices, []uint64{0}) {
} else if !reflect.DeepEqual(slices, []uint64{1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
@ -489,12 +487,12 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// The local node owns slice 1.
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
e := NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, (1 * SliceWidth) + 1, 2*SliceWidth + 4}) {
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected bits: %+v", bits)
}
}
@ -509,18 +507,18 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(10)}, nil
}
// Create local executor data. The local node owns slice 1.
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2)
e := NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(12) {
t.Fatalf("unexpected n: %d", res[0])
@ -539,9 +537,9 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(columnID=2, frame="f", id=10)` {
t.Fatalf("unexpected query: %s", query.String())
}
@ -554,17 +552,17 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
defer hldr.Close()
// Create frame.
if _, err := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil {
t.Fatal(err)
}
// Verify that one bit is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 {
if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
if !remoteCalled {
@ -584,9 +582,9 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != `i` {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `SetBit(columnID=2, frame="f", id=10, timestamp="2016-12-11T10:09")` {
t.Fatalf("unexpected query: %s", query.String())
}
@ -599,19 +597,19 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
defer hldr.Close()
// Create frame.
if f, err := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := f.SetTimeQuantum("Y"); err != nil {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
t.Fatal(err)
}
// Verify that one bit is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Row(10).Count(); n != 1 {
if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
if !remoteCalled {
@ -630,10 +628,10 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Mock secondary server's executor to verify arguments and return a bitmap.
var remoteExecN int
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if !reflect.DeepEqual(slices, []uint64{0, 2}) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "i" {
t.Fatalf("unexpected index: %s", index)
} else if !reflect.DeepEqual(slices, []uint64{1, 3}) {
t.Fatalf("unexpected slices: %+v", slices)
}
@ -661,14 +659,14 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
}}, nil
}
// Create local executor data on slice 1 & 3.
// Create local executor data on slice 2 & 4.
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
e := NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},

View file

@ -50,12 +50,12 @@ const (
DefaultFragmentMaxOpN = 2000
)
// Fragment represents the intersection of a frame and slice in a database.
// Fragment represents the intersection of a frame and slice in an index.
type Fragment struct {
mu sync.Mutex
// Composite identifiers
db string
index string
frame string
view string
slice uint64
@ -94,10 +94,10 @@ type Fragment struct {
}
// NewFragment returns a new instance of Fragment.
func NewFragment(path, db, frame, view string, slice uint64) *Fragment {
func NewFragment(path, index, frame, view string, slice uint64) *Fragment {
return &Fragment{
path: path,
db: db,
index: index,
frame: frame,
view: view,
slice: slice,
@ -117,8 +117,8 @@ func (f *Fragment) Path() string { return f.path }
// CachePath returns the path to the fragment's cache data.
func (f *Fragment) CachePath() string { return f.path + CacheExt }
// DB returns the database the fragment was initialized with.
func (f *Fragment) DB() string { return f.db }
// Index returns the index that the fragment was initialized with.
func (f *Fragment) Index() string { return f.index }
// Frame returns the frame the fragment was initialized with.
func (f *Fragment) Frame() string { return f.frame }
@ -1002,8 +1002,8 @@ func track(start time.Time, name string, logger *log.Logger) {
func (f *Fragment) snapshot() error {
logger := f.logger()
logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.db, f.frame, f.view, f.slice)
defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.db, f.frame, f.view, f.slice), logger)
logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice), logger)
// Create a temporary file to snapshot to.
snapshotPath := f.path + SnapshotExt
@ -1307,7 +1307,7 @@ func (s *FragmentSyncer) isClosing() bool {
// then merges any blocks which have differences.
func (s *FragmentSyncer) SyncFragment() error {
// Determine replica set.
nodes := s.Cluster.FragmentNodes(s.Fragment.DB(), s.Fragment.Slice())
nodes := s.Cluster.FragmentNodes(s.Fragment.Index(), s.Fragment.Slice())
if len(nodes) == 1 {
return nil
}
@ -1327,7 +1327,7 @@ func (s *FragmentSyncer) SyncFragment() error {
if err != nil {
return err
}
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
if err != nil && err != ErrFragmentNotFound {
return err
}
@ -1392,7 +1392,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
// Read pairs from each remote block.
var pairSets []PairSet
var clients []*Client
for _, node := range s.Cluster.FragmentNodes(f.DB(), f.Slice()) {
for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) {
if s.Host == node.Host {
continue
}
@ -1409,7 +1409,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
clients = append(clients, client)
// Only sync the standard block.
rowIDs, columnIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id)
rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), ViewStandard, f.Slice(), id)
if err != nil {
return err
}
@ -1457,7 +1457,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
}
// Execute query.
_, err := clients[i].ExecuteQuery(context.Background(), f.DB(), buf.String(), false)
_, err := clients[i].ExecuteQuery(context.Background(), f.Index(), buf.String(), false)
if err != nil {
return err
}

View file

@ -23,7 +23,7 @@ const SliceWidth = pilosa.SliceWidth
// Ensure a fragment can set a bit and retrieve it.
func TestFragment_SetBit(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the fragment.
@ -54,7 +54,7 @@ func TestFragment_SetBit(t *testing.T) {
// Ensure a fragment can clear a set bit.
func TestFragment_ClearBit(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set and then clear bits on the fragment.
@ -81,7 +81,7 @@ func TestFragment_ClearBit(t *testing.T) {
// Ensure a fragment can snapshot correctly.
func TestFragment_Snapshot(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set and then clear bits on the fragment.
@ -110,7 +110,7 @@ func TestFragment_Snapshot(t *testing.T) {
// Ensure a fragment can iterate over all bits in order.
func TestFragment_ForEachBit(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the fragment.
@ -139,7 +139,7 @@ func TestFragment_ForEachBit(t *testing.T) {
// Ensure a fragment can return the top n results.
func TestFragment_Top(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the rows 100, 101, & 102.
@ -161,7 +161,7 @@ func TestFragment_Top(t *testing.T) {
// Ensure a fragment can filter rows when retrieving the top n rows.
func TestFragment_Top_Filter(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the rows 100, 101, & 102.
@ -191,7 +191,7 @@ func TestFragment_Top_Filter(t *testing.T) {
// Ensure a fragment can return top rows that intersect with an input row.
func TestFragment_TopN_Intersect(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Create an intersecting input row.
@ -221,7 +221,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
t.Skip("short mode")
}
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Create an intersecting input row.
@ -258,7 +258,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
// Ensure a fragment can return top rows when specified by ID.
func TestFragment_TopN_IDs(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on various rows.
@ -282,12 +282,12 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
slice := uint64(0)
cacheSize := uint32(3)
// Create DB.
db := MustOpenDB()
defer db.Close()
// Create Index.
index := MustOpenIndex()
defer index.Close()
// Create frame.
frame, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize})
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize})
if err != nil {
t.Fatal(err)
}
@ -346,7 +346,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
// Ensure fragment can return a checksum for its blocks.
func TestFragment_Checksum(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Retrieve checksum and set bits.
@ -365,7 +365,7 @@ func TestFragment_Checksum(t *testing.T) {
// Ensure fragment can return a checksum for a given block.
func TestFragment_Blocks(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Retrieve initial checksum.
@ -403,7 +403,7 @@ func TestFragment_Blocks(t *testing.T) {
// Ensure fragment returns an empty checksum if no data exists for a block.
func TestFragment_Blocks_Empty(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on a different block.
@ -421,7 +421,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_LRUCache_Persistence(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the fragment.
@ -453,11 +453,11 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_RankCache_Persistence(t *testing.T) {
db := MustOpenDB()
defer db.Close()
index := MustOpenIndex()
defer index.Close()
// Create frame.
frame, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
if err != nil {
t.Fatal(err)
}
@ -488,13 +488,13 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
t.Fatalf("unexpected cache len: %d", cache.Len())
}
// Reopen the database.
if err := db.Reopen(); err != nil {
// Reopen the index.
if err := index.Reopen(); err != nil {
t.Fatal(err)
}
// Re-fetch fragment.
f = db.Frame("f").View(pilosa.ViewStandard).Fragment(0)
f = index.Frame("f").View(pilosa.ViewStandard).Fragment(0)
// Re-verify correct cache type and size.
if cache, ok := f.Cache().(*pilosa.RankCache); !ok {
@ -506,7 +506,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
// Ensure a fragment can be copied to another fragment.
func TestFragment_WriteTo_ReadFrom(t *testing.T) {
f0 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f0 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f0.Close()
// Set and then clear bits on the fragment.
@ -531,7 +531,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
}
// Read into another fragment.
f1 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f1 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
if rn, err := f1.ReadFrom(&buf); err != nil {
t.Fatal(err)
} else if wn != rn {
@ -564,7 +564,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
}
// Open the fragment specified by the path.
f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0)
f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0)
if err := f.Open(); err != nil {
b.Fatal(err)
}
@ -580,7 +580,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
}
func BenchmarkFragment_IntersectionCount(b *testing.B) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
f.MaxOpN = math.MaxInt32
@ -617,7 +617,7 @@ type Fragment struct {
}
// NewFragment returns a new instance of Fragment with a temporary path.
func NewFragment(db, frame, view string, slice uint64) *Fragment {
func NewFragment(index, frame, view string, slice uint64) *Fragment {
file, err := ioutil.TempFile("", "pilosa-fragment-")
if err != nil {
panic(err)
@ -625,7 +625,7 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment {
file.Close()
f := &Fragment{
Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice),
Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice),
RowAttrStore: MustOpenAttrStore(),
}
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
@ -633,8 +633,8 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment {
}
// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error.
func MustOpenFragment(db, frame, view string, slice uint64) *Fragment {
f := NewFragment(db, frame, view, slice)
func MustOpenFragment(index, frame, view string, slice uint64) *Fragment {
f := NewFragment(index, frame, view, slice)
if err := f.Open(); err != nil {
panic(err)
}
@ -656,7 +656,7 @@ func (f *Fragment) Reopen() error {
return err
}
f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice())
f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice())
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
if err := f.Open(); err != nil {
return err
@ -720,7 +720,7 @@ func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
}
func TestFragment_Tanimoto(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
src := pilosa.NewBitmap(1, 2, 3)
@ -742,7 +742,7 @@ func TestFragment_Tanimoto(t *testing.T) {
}
func TestFragment_Zero_Tanimoto(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
defer f.Close()
src := pilosa.NewBitmap(1, 2, 3)

View file

@ -29,7 +29,7 @@ const (
type Frame struct {
mu sync.Mutex
path string
db string
index string
name string
timeQuantum TimeQuantum
@ -53,16 +53,16 @@ type Frame struct {
}
// NewFrame returns a new instance of frame.
func NewFrame(path, db, name string) (*Frame, error) {
func NewFrame(path, index, name string) (*Frame, error) {
err := ValidateName(name)
if err != nil {
return nil, err
}
return &Frame{
path: path,
db: db,
name: name,
path: path,
index: index,
name: name,
views: make(map[string]*View),
rowAttrStore: NewAttrStore(filepath.Join(path, ".data")),
@ -81,8 +81,8 @@ func NewFrame(path, db, name string) (*Frame, error) {
// Name returns the name the frame was initialized with.
func (f *Frame) Name() string { return f.name }
// DB returns the database name the frame was initialized with.
func (f *Frame) DB() string { return f.db }
// Index returns the index name the frame was initialized with.
func (f *Frame) Index() string { return f.index }
// Path returns the path the frame was initialized with.
func (f *Frame) Path() string { return f.path }
@ -418,7 +418,7 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
}
func (f *Frame) newView(path, name string) *View {
view := NewView(path, f.db, f.name, name, f.cacheSize)
view := NewView(path, f.index, f.name, name, f.cacheSize)
view.cacheType = f.cacheType
view.LogOutput = f.LogOutput
view.RowAttrStore = f.rowAttrStore
@ -515,7 +515,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
// Determine quantum if timestamps are set.
q := f.TimeQuantum()
if hasTime(timestamps) && q == "" {
return errors.New("time quantum not set in either database or frame")
return errors.New("time quantum not set in either index or frame")
}
// Split import data by fragment.

View file

@ -60,7 +60,7 @@ func TestFrame_NameRestriction(t *testing.T) {
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "d", ".meta")
frame, err := pilosa.NewFrame(path, "i", ".meta")
if frame != nil {
t.Fatalf("unexpected frame name %s", err)
}
@ -77,7 +77,7 @@ func NewFrame() *Frame {
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "d", "f")
frame, err := pilosa.NewFrame(path, "i", "f")
if err != nil {
panic(err)
}
@ -99,15 +99,15 @@ func (f *Frame) Close() error {
return f.Frame.Close()
}
// Reopen closes the database and reopens it.
// Reopen closes the index and reopens it.
func (f *Frame) Reopen() error {
var err error
if err := f.Frame.Close(); err != nil {
return err
}
path, db, name := f.Path(), f.DB(), f.Name()
f.Frame, err = pilosa.NewFrame(path, db, name)
path, index, name := f.Path(), f.Index(), f.Name()
f.Frame, err = pilosa.NewFrame(path, index, name)
if err != nil {
return err
}

View file

@ -38,7 +38,7 @@ type Handler struct {
// The execution engine for running queries.
Executor interface {
Execute(context context.Context, db string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error)
Execute(context context.Context, index string, query *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error)
}
// The version to report on the /version endpoint.
@ -59,20 +59,20 @@ func NewHandler() *Handler {
func NewRouter(handler *Handler) *mux.Router {
router := mux.NewRouter()
router.HandleFunc("/db", handler.handleGetDBs).Methods("GET")
router.HandleFunc("/db/{db}", handler.handleGetDB).Methods("GET")
router.HandleFunc("/db/{db}", handler.handlePostDB).Methods("POST")
router.HandleFunc("/db/{db}", handler.handleDeleteDB).Methods("DELETE")
router.HandleFunc("/db/{db}/attr/diff", handler.handlePostDBAttrDiff).Methods("POST")
//router.HandleFunc("/db/{db}/frame", handler.handleGetFrames).Methods("GET") // Not implemented.
router.HandleFunc("/db/{db}/frame/{frame}", handler.handlePostFrame).Methods("POST")
router.HandleFunc("/db/{db}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE")
router.HandleFunc("/db/{db}/query", handler.handlePostQuery).Methods("POST")
router.HandleFunc("/db/{db}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST")
router.HandleFunc("/db/{db}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST")
router.HandleFunc("/db/{db}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH")
router.HandleFunc("/db/{db}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET")
router.HandleFunc("/db/{db}/time-quantum", handler.handlePatchDBTimeQuantum).Methods("PATCH")
router.HandleFunc("/index", handler.handleGetIndexes).Methods("GET")
router.HandleFunc("/index/{index}", handler.handleGetIndex).Methods("GET")
router.HandleFunc("/index/{index}", handler.handlePostIndex).Methods("POST")
router.HandleFunc("/index/{index}", handler.handleDeleteIndex).Methods("DELETE")
router.HandleFunc("/index/{index}/attr/diff", handler.handlePostIndexAttrDiff).Methods("POST")
//router.HandleFunc("/index/{index}/frame", handler.handleGetFrames).Methods("GET") // Not implemented.
router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE")
router.HandleFunc("/index/{index}/query", handler.handlePostQuery).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/time-quantum", handler.handlePatchFrameTimeQuantum).Methods("PATCH")
router.HandleFunc("/index/{index}/frame/{frame}/views", handler.handleGetFrameViews).Methods("GET")
router.HandleFunc("/index/{index}/time-quantum", handler.handlePatchIndexTimeQuantum).Methods("PATCH")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.HandleFunc("/debug/vars", handler.handleExpvar).Methods("GET")
router.HandleFunc("/export", handler.handleGetExport).Methods("GET")
@ -91,7 +91,7 @@ func NewRouter(handler *Handler) *mux.Router {
// Ideally this would be automatic, as described in this (wontfix) ticket:
// https://github.com/gorilla/mux/issues/6
// For now we just do it for the most commonly used handler, /query
router.HandleFunc("/db/{db}/query", handler.methodNotAllowedHandler).Methods("GET")
router.HandleFunc("/index/{index}/query", handler.methodNotAllowedHandler).Methods("GET")
return router
}
@ -108,7 +108,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// handleGetSchema handles GET /schema requests.
func (h *Handler) handleGetSchema(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(getSchemaResponse{
DBs: h.Holder.Schema(),
Indexes: h.Holder.Schema(),
}); err != nil {
h.logger().Printf("write schema response error: %s", err)
}
@ -124,7 +124,7 @@ func (h *Handler) handleGetStatus(w http.ResponseWriter, r *http.Request) {
}
type getSchemaResponse struct {
DBs []*DBInfo `json:"dbs"`
Indexes []*IndexInfo `json:"indexes"`
}
type getStatusResponse struct {
@ -133,7 +133,7 @@ type getStatusResponse struct {
// handlePostQuery handles /query requests.
func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
indexName := mux.Vars(r)["index"]
// Parse incoming request.
req, err := h.readQueryRequest(r)
@ -157,7 +157,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
}
// Execute the query.
results, err := h.Executor.Execute(r.Context(), dbName, q, req.Slices, opt)
results, err := h.Executor.Execute(r.Context(), indexName, q, req.Slices, opt)
resp := &QueryResponse{Results: results, Err: err}
// Fill column attributes if requested.
@ -173,7 +173,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
}
// Retrieve column attributes across all calls.
columnAttrSets, err := h.readColumnAttrSets(h.Holder.DB(dbName), columnIDs)
columnAttrSets, err := h.readColumnAttrSets(h.Holder.Index(indexName), columnIDs)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
@ -219,40 +219,40 @@ type sliceMaxResponse struct {
MaxSlices map[string]uint64 `json:"maxSlices"`
}
// handleGetDBs handles GET /db request.
func (h *Handler) handleGetDBs(w http.ResponseWriter, r *http.Request) {
// handleGetIndexes handles GET /index request.
func (h *Handler) handleGetIndexes(w http.ResponseWriter, r *http.Request) {
h.handleGetSchema(w, r)
}
// handleGetDB handles GET /db/<dbname> requests.
func (h *Handler) handleGetDB(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
db := h.Holder.DB(dbName)
if db == nil {
http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
// handleGetIndex handles GET /index/<indexname> requests.
func (h *Handler) handleGetIndex(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
index := h.Holder.Index(indexName)
if index == nil {
http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound)
return
}
if err := json.NewEncoder(w).Encode(getDBResponse{
map[string]string{"name": db.Name()},
if err := json.NewEncoder(w).Encode(getIndexResponse{
map[string]string{"name": index.Name()},
}); err != nil {
h.logger().Printf("write response error: %s", err)
}
}
type getDBResponse struct {
DB map[string]string `json:"db"`
type getIndexResponse struct {
Index map[string]string `json:"index"`
}
type postDBRequest struct {
Options DBOptions `json:"options"`
type postIndexRequest struct {
Options IndexOptions `json:"options"`
}
//_postDBRequest is necessary to avoid recursion while decoding.
type _postDBRequest postDBRequest
//_postIndexRequest is necessary to avoid recursion while decoding.
type _postIndexRequest postIndexRequest
// Custom Unmarshal JSON to validate request body when creating a new database
func (p *postDBRequest) UnmarshalJSON(b []byte) error {
// Custom Unmarshal JSON to validate request body when creating a new index.
func (p *postIndexRequest) UnmarshalJSON(b []byte) error {
// m is an overflow map used to capture additional, unexpected keys.
m := make(map[string]interface{})
@ -260,13 +260,13 @@ func (p *postDBRequest) UnmarshalJSON(b []byte) error {
return err
}
validDBOptions := getValidOptions(DBOptions{})
err := validateOptions(m, validDBOptions)
validIndexOptions := getValidOptions(IndexOptions{})
err := validateOptions(m, validIndexOptions)
if err != nil {
return err
}
// Unmarshal expected values.
var _p _postDBRequest
var _p _postIndexRequest
if err := json.Unmarshal(b, &_p); err != nil {
return err
}
@ -277,7 +277,7 @@ func (p *postDBRequest) UnmarshalJSON(b []byte) error {
}
// Raise errors for any unknown key
func validateOptions(data map[string]interface{}, validDBOptions []string) error {
func validateOptions(data map[string]interface{}, validIndexOptions []string) error {
for k, v := range data {
switch k {
case "options":
@ -286,7 +286,7 @@ func validateOptions(data map[string]interface{}, validDBOptions []string) error
return errors.New("options is not map[string]interface{}")
}
for kk, vv := range options {
if !foundItem(validDBOptions, kk) {
if !foundItem(validIndexOptions, kk) {
return fmt.Errorf("Unknown key: %v:%v", kk, vv)
}
}
@ -306,53 +306,53 @@ func foundItem(items []string, item string) bool {
return false
}
type postDBResponse struct{}
type postIndexResponse struct{}
// handleDeleteDB handles DELETE /db request.
func (h *Handler) handleDeleteDB(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
// handleDeleteIndex handles DELETE /index request.
func (h *Handler) handleDeleteIndex(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
// Delete database from the holder.
if err := h.Holder.DeleteDB(dbName); err != nil {
// Delete index from the holder.
if err := h.Holder.DeleteIndex(indexName); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Send the delete database message to all nodes.
// Send the delete index message to all nodes.
err := h.Broadcaster.SendSync(
&internal.DeleteDBMessage{
DB: dbName,
&internal.DeleteIndexMessage{
Index: indexName,
})
if err != nil {
h.logger().Printf("problem sending DeleteDB message: %s", err)
h.logger().Printf("problem sending DeleteIndex message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil {
if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type deleteDBResponse struct{}
type deleteIndexResponse struct{}
// handlePostDB handles POST /db request.
func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
// handlePostIndex handles POST /index request.
func (h *Handler) handlePostIndex(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
// Decode request.
var req postDBRequest
var req postIndexRequest
err := json.NewDecoder(r.Body).Decode(&req)
if err == io.EOF {
// If no data was provided (EOF), we still create the database
// If no data was provided (EOF), we still create the index
// with default values.
} else if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Create database.
_, err = h.Holder.CreateDB(dbName, req.Options)
if err == ErrDatabaseExists {
// Create index.
_, err = h.Holder.CreateIndex(indexName, req.Options)
if err == ErrIndexExists {
http.Error(w, err.Error(), http.StatusConflict)
return
} else if err != nil {
@ -360,28 +360,28 @@ func (h *Handler) handlePostDB(w http.ResponseWriter, r *http.Request) {
return
}
// Send the create database message to all nodes.
// Send the create index message to all nodes.
err = h.Broadcaster.SendSync(
&internal.CreateDBMessage{
DB: dbName,
Meta: req.Options.Encode(),
&internal.CreateIndexMessage{
Index: indexName,
Meta: req.Options.Encode(),
})
if err != nil {
h.logger().Printf("problem sending CreateDB message: %s", err)
h.logger().Printf("problem sending CreateIndex message: %s", err)
}
// Encode response.
if err := json.NewEncoder(w).Encode(postDBResponse{}); err != nil {
if err := json.NewEncoder(w).Encode(postIndexResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
// handlePatchDBTimeQuantum handles PATCH /db/time_quantum request.
func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
// handlePatchIndexTimeQuantum handles PATCH /index/time_quantum request.
func (h *Handler) handlePatchIndexTimeQuantum(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
// Decode request.
var req patchDBTimeQuantumRequest
var req patchIndexTimeQuantumRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@ -394,51 +394,51 @@ func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Reques
return
}
// Retrieve database by name.
database := h.Holder.DB(dbName)
if database == nil {
http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
// Retrieve index by name.
index := h.Holder.Index(indexName)
if index == nil {
http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound)
return
}
// Set default time quantum on database.
if err := database.SetTimeQuantum(tq); err != nil {
// Set default time quantum on index.
if err := index.SetTimeQuantum(tq); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Encode response.
if err := json.NewEncoder(w).Encode(patchDBTimeQuantumResponse{}); err != nil {
if err := json.NewEncoder(w).Encode(patchIndexTimeQuantumResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type patchDBTimeQuantumRequest struct {
type patchIndexTimeQuantumRequest struct {
TimeQuantum string `json:"timeQuantum"`
}
type patchDBTimeQuantumResponse struct{}
type patchIndexTimeQuantumResponse struct{}
// handlePostDBAttrDiff handles POST /db/attr/diff requests.
func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
// handlePostIndexAttrDiff handles POST /index/attr/diff requests.
func (h *Handler) handlePostIndexAttrDiff(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
// Decode request.
var req postDBAttrDiffRequest
var req postIndexAttrDiffRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// Retrieve database from holder.
db := h.Holder.DB(dbName)
if db == nil {
http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
// Retrieve index from holder.
index := h.Holder.Index(indexName)
if index == nil {
http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound)
return
}
// Retrieve local blocks.
blks, err := db.ColumnAttrStore().Blocks()
blks, err := index.ColumnAttrStore().Blocks()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -448,37 +448,37 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) {
attrs := make(map[uint64]map[string]interface{})
for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) {
// Retrieve block data.
m, err := db.ColumnAttrStore().BlockData(blockID)
m, err := index.ColumnAttrStore().BlockData(blockID)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
// Copy to database-wide struct.
// Copy to index-wide struct.
for k, v := range m {
attrs[k] = v
}
}
// Encode response.
if err := json.NewEncoder(w).Encode(postDBAttrDiffResponse{
if err := json.NewEncoder(w).Encode(postIndexAttrDiffResponse{
Attrs: attrs,
}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
}
type postDBAttrDiffRequest struct {
type postIndexAttrDiffRequest struct {
Blocks []AttrBlock `json:"blocks"`
}
type postDBAttrDiffResponse struct {
type postIndexAttrDiffResponse struct {
Attrs map[uint64]map[string]interface{} `json:"attrs"`
}
// handlePostFrame handles POST /frame request.
func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
// Decode request.
@ -492,15 +492,15 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
return
}
// Find database.
db := h.Holder.DB(dbName)
if db == nil {
http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
// Find index.
index := h.Holder.Index(indexName)
if index == nil {
http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound)
return
}
// Create frame.
_, err = db.CreateFrame(frameName, req.Options)
_, err = index.CreateFrame(frameName, req.Options)
if err == ErrFrameExists {
http.Error(w, err.Error(), http.StatusConflict)
return
@ -512,7 +512,7 @@ func (h *Handler) handlePostFrame(w http.ResponseWriter, r *http.Request) {
// Send the create frame message to all nodes.
err = h.Broadcaster.SendSync(
&internal.CreateFrameMessage{
DB: dbName,
Index: indexName,
Frame: frameName,
Meta: req.Options.Encode(),
})
@ -573,20 +573,20 @@ type postFrameResponse struct{}
// handleDeleteFrame handles DELETE /frame request.
func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
// Find database.
db := h.Holder.DB(dbName)
if db == nil {
if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil {
// Find index.
index := h.Holder.Index(indexName)
if index == nil {
if err := json.NewEncoder(w).Encode(deleteIndexResponse{}); err != nil {
h.logger().Printf("response encoding error: %s", err)
}
return
}
// Delete frame from the database.
if err := db.DeleteFrame(frameName); err != nil {
// Delete frame from the index.
if err := index.DeleteFrame(frameName); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@ -594,7 +594,7 @@ func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) {
// Send the delete frame message to all nodes.
err := h.Broadcaster.SendSync(
&internal.DeleteFrameMessage{
DB: dbName,
Index: indexName,
Frame: frameName,
})
if err != nil {
@ -611,7 +611,7 @@ type deleteFrameResponse struct{}
// handlePatchFrameTimeQuantum handles PATCH /frame/time_quantum request.
func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
// Decode request.
@ -628,14 +628,14 @@ func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Req
return
}
// Retrieve database by name.
f := h.Holder.Frame(dbName, frameName)
// Retrieve index by name.
f := h.Holder.Frame(indexName, frameName)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
// Set default time quantum on database.
// Set default time quantum on index.
if err := f.SetTimeQuantum(tq); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -655,11 +655,11 @@ type patchFrameTimeQuantumResponse struct{}
// handleGetFrameViews handles GET /frame/views request.
func (h *Handler) handleGetFrameViews(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
// Retrieve views.
f := h.Holder.Frame(dbName, frameName)
f := h.Holder.Frame(indexName, frameName)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
@ -684,7 +684,7 @@ type getFrameViewsResponse struct {
// handlePostFrameAttrDiff handles POST /frame/attr/diff requests.
func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
// Decode request.
@ -694,8 +694,8 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request
return
}
// Retrieve database from holder.
f := h.Holder.Frame(dbName, frameName)
// Retrieve index from holder.
f := h.Holder.Frame(indexName, frameName)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
@ -718,7 +718,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request
return
}
// Copy to database-wide struct.
// Copy to index-wide struct.
for k, v := range m {
attrs[k] = v
}
@ -741,15 +741,15 @@ type postFrameAttrDiffResponse struct {
}
// readColumnAttrSets returns a list of column attribute objects by id.
func (h *Handler) readColumnAttrSets(db *DB, ids []uint64) ([]*ColumnAttrSet, error) {
if db == nil {
func (h *Handler) readColumnAttrSets(index *Index, ids []uint64) ([]*ColumnAttrSet, error) {
if index == nil {
return nil, nil
}
a := make([]*ColumnAttrSet, 0, len(ids))
for _, id := range ids {
// Read attributes for column. Skip column if empty.
attrs, err := db.ColumnAttrStore().Attrs(id)
attrs, err := index.ColumnAttrStore().Attrs(id)
if err != nil {
return nil, err
} else if len(attrs) == 0 {
@ -884,25 +884,25 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
// Validate that this handler owns the slice.
if !h.Cluster.OwnsFragment(h.Host, req.DB, req.Slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, req.DB, req.Slice)
if !h.Cluster.OwnsFragment(h.Host, req.Index, req.Slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, req.Index, req.Slice)
http.Error(w, mesg, http.StatusPreconditionFailed)
return
}
// Find the DB.
h.logger().Println("importing:", req.DB, req.Frame, req.Slice)
db := h.Holder.DB(req.DB)
if db == nil {
h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrDatabaseNotFound.Error())
http.Error(w, ErrDatabaseNotFound.Error(), http.StatusNotFound)
// Find the Index.
h.logger().Println("importing:", req.Index, req.Frame, req.Slice)
index := h.Holder.Index(req.Index)
if index == nil {
h.logger().Printf("fragment error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrIndexNotFound.Error())
http.Error(w, ErrIndexNotFound.Error(), http.StatusNotFound)
return
}
// Retrieve frame.
f := db.Frame(req.Frame)
f := index.Frame(req.Frame)
if f == nil {
h.logger().Printf("frame error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, ErrFrameNotFound.Error())
h.logger().Printf("frame error: index=%s, frame=%s, slice=%d, err=%s", req.Index, req.Frame, req.Slice, ErrFrameNotFound.Error())
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
@ -910,7 +910,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
// Import into fragment.
err = f.Import(req.RowIDs, req.ColumnIDs, timestamps)
if err != nil {
h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ColumnIDs), err)
h.logger().Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
return
}
@ -941,7 +941,7 @@ func (h *Handler) handleGetExport(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
// Parse query parameters.
q := r.URL.Query()
db, frame, view := q.Get("db"), q.Get("frame"), q.Get("view")
index, frame, view := q.Get("index"), q.Get("frame"), q.Get("view")
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
if err != nil {
@ -950,14 +950,14 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
}
// Validate that this handler owns the slice.
if !h.Cluster.OwnsFragment(h.Host, db, slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, db, slice)
if !h.Cluster.OwnsFragment(h.Host, index, slice) {
mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, index, slice)
http.Error(w, mesg, http.StatusPreconditionFailed)
return
}
// Find the fragment.
f := h.Holder.Fragment(db, frame, view, slice)
f := h.Holder.Fragment(index, frame, view, slice)
if f == nil {
return
}
@ -983,7 +983,7 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
// handleGetFragmentNodes handles /fragment/nodes requests.
func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) {
q := r.URL.Query()
db := q.Get("db")
index := q.Get("index")
// Read slice parameter.
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
@ -993,7 +993,7 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
}
// Retrieve fragment owner nodes.
nodes := h.Cluster.FragmentNodes(db, slice)
nodes := h.Cluster.FragmentNodes(index, slice)
// Write to response.
if err := json.NewEncoder(w).Encode(nodes); err != nil {
@ -1012,7 +1012,7 @@ func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request)
}
// Retrieve fragment from holder.
f := h.Holder.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice)
f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice)
if f == nil {
http.Error(w, "fragment not found", http.StatusNotFound)
return
@ -1035,7 +1035,7 @@ func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request)
}
// Retrieve frame.
f := h.Holder.Frame(q.Get("db"), q.Get("frame"))
f := h.Holder.Frame(q.Get("index"), q.Get("frame"))
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
@ -1075,7 +1075,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ
}
// Retrieve fragment from holder.
f := h.Holder.Fragment(req.DB, req.Frame, req.View, req.Slice)
f := h.Holder.Fragment(req.Index, req.Frame, req.View, req.Slice)
if f == nil {
http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound)
return
@ -1111,7 +1111,7 @@ func (h *Handler) handleGetFragmentBlocks(w http.ResponseWriter, r *http.Request
}
// Retrieve fragment from holder.
f := h.Holder.Fragment(q.Get("db"), q.Get("frame"), q.Get("view"), slice)
f := h.Holder.Fragment(q.Get("index"), q.Get("frame"), q.Get("view"), slice)
if f == nil {
http.Error(w, "fragment not found", http.StatusNotFound)
return
@ -1134,7 +1134,7 @@ type getFragmentBlocksResponse struct {
// handlePostFrameRestore handles POST /frame/restore requests.
func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) {
dbName := mux.Vars(r)["db"]
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
q := r.URL.Query()
@ -1154,30 +1154,30 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
}
// Determine the maximum number of slices.
maxSlices, err := client.MaxSliceByDatabase(r.Context())
maxSlices, err := client.MaxSliceByIndex(r.Context())
if err != nil {
http.Error(w, "cannot determine remote slice count: "+err.Error(), http.StatusInternalServerError)
return
}
// Retrieve frame.
f := h.Holder.Frame(dbName, frameName)
f := h.Holder.Frame(indexName, frameName)
if f == nil {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
return
}
// Retrieve list of all views.
views, err := client.FrameViews(r.Context(), dbName, frameName)
views, err := client.FrameViews(r.Context(), indexName, frameName)
if err != nil {
http.Error(w, "cannot retrieve frame views: "+err.Error(), http.StatusInternalServerError)
return
}
// Loop over each slice and import it if this node owns it.
for slice := uint64(0); slice <= maxSlices[dbName]; slice++ {
for slice := uint64(0); slice <= maxSlices[indexName]; slice++ {
// Ignore this slice if we don't own it.
if !h.Cluster.OwnsFragment(h.Host, dbName, slice) {
if !h.Cluster.OwnsFragment(h.Host, indexName, slice) {
continue
}
@ -1198,7 +1198,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request)
}
// Stream backup from remote node.
rd, err := client.BackupSlice(r.Context(), dbName, frameName, view, slice)
rd, err := client.BackupSlice(r.Context(), indexName, frameName, view, slice)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -1262,8 +1262,8 @@ func (h *Handler) logger() *log.Logger {
// QueryRequest represent a request to process a query.
type QueryRequest struct {
// Database to execute query against.
DB string
// Index to execute query against.
Index string
// The query string to parse and execute.
Query string

View file

@ -6,21 +6,21 @@ import (
"testing"
)
// Test custom UnmarshalJSON for postDBRequest object
func TestPostDBRequestUnmarshalJSON(t *testing.T) {
// Test custom UnmarshalJSON for postIndexRequest object
func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
tests := []struct {
json string
expected postDBRequest
expected postIndexRequest
err string
}{
{json: `{"options": {}}`, expected: postDBRequest{Options: DBOptions{}}},
{json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}},
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"columnLabel": "test"}}`, expected: postDBRequest{Options: DBOptions{ColumnLabel: "test"}}},
{json: `{"options": {"columnLabel": "test"}}`, expected: postIndexRequest{Options: IndexOptions{ColumnLabel: "test"}}},
{json: `{"options": {"columnLabl": "test"}}`, err: "Unknown key: columnLabl:test"},
}
for _, test := range tests {
actual := &postDBRequest{}
actual := &postIndexRequest{}
err := json.Unmarshal([]byte(test.json), actual)
if err != nil {

View file

@ -34,22 +34,22 @@ func TestHandler_Schema(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
d0 := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
d1 := hldr.MustCreateDBIfNotExists("d1", pilosa.DBOptions{})
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := d0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -59,7 +59,7 @@ func TestHandler_Schema(t *testing.T) {
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"dbs":[{"name":"d0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"d1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -69,13 +69,13 @@ func TestHandler_MaxSlices(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4)
hldr.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
h := NewHandler()
h.Holder = hldr.Holder
@ -83,7 +83,7 @@ func TestHandler_MaxSlices(t *testing.T) {
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -93,7 +93,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
f0, err := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true})
f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true})
if err != nil {
t.Fatal(err)
}
@ -105,7 +105,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
t.Fatal(err)
}
f1, err := hldr.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true})
f1, err := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true})
if err != nil {
t.Fatal(err)
}
@ -123,7 +123,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -131,9 +131,9 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Count(Bitmap(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
@ -143,7 +143,7 @@ func TestHandler_Query_Args_URL(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
@ -154,9 +154,9 @@ func TestHandler_Query_Args_URL(t *testing.T) {
// Ensure the handler can accept arguments via protobufs.
func TestHandler_Query_Args_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Count(Bitmap(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
@ -175,7 +175,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
}
// Generate protobuf request.
req := MustNewHTTPRequest("POST", "/db/db0/query", bytes.NewReader(reqBody))
req := MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/x-protobuf")
w := httptest.NewRecorder()
@ -188,7 +188,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
// Ensure the handler returns an error when parsing bad arguments.
func TestHandler_Query_Args_Err(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
@ -199,12 +199,12 @@ func TestHandler_Query_Args_Err(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as JSON.
func TestHandler_Query_Uint64_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
@ -215,12 +215,12 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as protobufs.
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader("Count(Bitmap(id=100))"))
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
@ -238,14 +238,14 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap as JSON.
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return []interface{}{bm}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader("Bitmap(id=100)")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" {
@ -258,26 +258,26 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
hldr := NewHolder()
defer hldr.Close()
// Create database and set column attributes.
db, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{})
// Create index and set column attributes.
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
} else if err := db.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
} else if err := index.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := db.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil {
} else if err := index.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil {
t.Fatal(err)
}
h := NewHandler()
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return []interface{}{bm}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
@ -288,14 +288,14 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap as protobuf.
func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return []interface{}{bm}, nil
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader("Bitmap(id=100)"))
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
@ -323,17 +323,17 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
hldr := NewHolder()
defer hldr.Close()
// Create database and set column attributes.
db, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{})
// Create index and set column attributes.
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
} else if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
} else if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
}
h := NewHandler()
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return []interface{}{bm}, nil
@ -349,7 +349,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/db/d/query", bytes.NewReader(buf))
r := MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf))
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
@ -387,7 +387,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as JSON.
func TestHandler_Query_Pairs_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
{ID: 3, Count: 4},
@ -395,7 +395,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`TopN(frame=x, n=2)`)))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" {
@ -406,7 +406,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as protobuf.
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
{ID: 3, Count: 4},
@ -414,7 +414,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`TopN(frame=x, n=2)`))
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
@ -432,12 +432,12 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
// Ensure the handler can return an error as JSON.
func TestHandler_Query_Err_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`Bitmap(id=100)`)))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`)))
if w.Code != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" {
@ -448,12 +448,12 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
// Ensure the handler can return an error as protobuf.
func TestHandler_Query_Err_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`TopN(frame=x, n=2)`))
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusInternalServerError {
@ -471,7 +471,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) {
// Ensure the handler returns "method not allowed" for non-POST queries.
func TestHandler_Query_MethodNotAllowed(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/db/d/query", nil))
NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i/query", nil))
if w.Code != http.StatusMethodNotAllowed {
t.Fatalf("invalid status: %d", w.Code)
}
@ -481,7 +481,7 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) {
func TestHandler_Query_ErrParse(t *testing.T) {
h := NewHandler()
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("bad_fn(")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" {
@ -489,8 +489,8 @@ func TestHandler_Query_ErrParse(t *testing.T) {
}
}
// Ensure the handler can delete a database.
func TestHandler_DB_Delete(t *testing.T) {
// Ensure the handler can delete an index.
func TestHandler_Index_Delete(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
@ -498,13 +498,13 @@ func TestHandler_DB_Delete(t *testing.T) {
s.Handler.Holder = hldr.Holder
defer s.Close()
// Create database.
if _, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{}); err != nil {
// Create index.
if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
// Send request to delete database.
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/db/d", strings.NewReader("")))
// Send request to delete index.
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
if err != nil {
t.Fatal(err)
}
@ -519,9 +519,9 @@ func TestHandler_DB_Delete(t *testing.T) {
t.Fatalf("unexpected response body: %s", buf)
}
// Verify database is gone.
if hldr.DB("d") != nil {
t.Fatal("expected nil database")
// Verify index is gone.
if hldr.Index("i") != nil {
t.Fatal("expected nil index")
}
}
@ -529,39 +529,39 @@ func TestHandler_DB_Delete(t *testing.T) {
func TestHandler_DeleteFrame(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
d0 := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
h := NewHandler()
h.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/db/d0/frame/f1", strings.NewReader("")))
h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader("")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.DB("d0").Frame("f1"); f != nil {
} else if f := hldr.Index("i0").Frame("f1"); f != nil {
t.Fatal("expected nil frame")
}
}
// Ensure handler can set the DB time quantum.
func TestHandler_SetDBTimeQuantum(t *testing.T) {
// Ensure handler can set the Index time quantum.
func TestHandler_SetIndexTimeQuantum(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
h := NewHandler()
h.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if q := hldr.DB("d0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
} else if q := hldr.Index("i0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected time quantum: %s", q)
}
}
@ -572,25 +572,25 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) {
defer hldr.Close()
// Create frame.
if _, err := hldr.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil {
if _, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
h := NewHandler()
h.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if q := hldr.DB("d0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
} else if q := hldr.Index("i0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected time quantum: %s", q)
}
}
// Ensure the handler can return data in differing blocks for a database.
func TestHandler_DB_AttrStore_Diff(t *testing.T) {
// Ensure the handler can return data in differing blocks for an index.
func TestHandler_Index_AttrStore_Diff(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
@ -598,21 +598,21 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) {
s.Handler.Holder = hldr.Holder
defer s.Close()
// Set attributes on the database.
db, err := hldr.CreateDBIfNotExists("d", pilosa.DBOptions{})
// Set attributes on the index.
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := db.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
} else if err := index.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := db.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
} else if err := index.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
// Retrieve block checksums.
blks, err := db.ColumnAttrStore().Blocks()
blks, err := index.ColumnAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
@ -623,7 +623,7 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) {
// Send block checksums to determine diff.
resp, err := http.Post(
s.URL+"/db/d/attr/diff",
s.URL+"/index/i/attr/diff",
"application/json",
strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`),
)
@ -647,8 +647,8 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
s.Handler.Holder = hldr.Holder
defer s.Close()
// Set attributes on the database.
d := hldr.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
// Set attributes on the index.
d := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := d.CreateFrameIfNotExists("meta", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
@ -673,7 +673,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
// Send block checksums to determine diff.
resp, err := http.Post(
s.URL+"/db/d/frame/meta/attr/diff",
s.URL+"/index/i/frame/meta/attr/diff",
"application/json",
strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`),
)
@ -698,11 +698,11 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
defer s.Close()
// Set bits in the index.
f0 := hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
f0.MustSetBits(100, 1, 2, 3)
// Begin backing up from slice d/f/0.
resp, err := http.Get(s.URL + "/fragment/data?db=d&frame=f&view=standard&slice=0")
resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0")
if err != nil {
t.Fatal(err)
}
@ -714,12 +714,12 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
}
// Create frame.
if _, err := hldr.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil {
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Restore backup to slice x/y/0.
if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil {
if resp, err := http.Post(s.URL+"/fragment/data?index=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
resp.Body.Close()
@ -759,7 +759,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
h.Cluster.ReplicaN = 2
w := httptest.NewRecorder()
r := MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
r := MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
@ -802,13 +802,13 @@ func NewHandler() *Handler {
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return c.ExecuteFn(ctx, db, query, slices, opt)
func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return c.ExecuteFn(ctx, index, query, slices, opt)
}
// Server represents a test wrapper for httptest.Server.

222
holder.go
View file

@ -20,8 +20,8 @@ const DefaultCacheFlushInterval = 1 * time.Minute
type Holder struct {
mu sync.Mutex
// Databases by name.
dbs map[string]*DB
// Indexes by name.
indexes map[string]*Index
Broadcaster Broadcaster
// Close management
@ -43,7 +43,7 @@ type Holder struct {
// NewHolder returns a new instance of Holder.
func NewHolder() *Holder {
return &Holder{
dbs: make(map[string]*DB),
indexes: make(map[string]*Index),
closing: make(chan struct{}, 0),
Stats: NopStatsClient,
@ -60,7 +60,7 @@ func (h *Holder) Open() error {
return err
}
// Open path to read all database directories.
// Open path to read all index directories.
f, err := os.Open(h.Path)
if err != nil {
return err
@ -77,25 +77,25 @@ func (h *Holder) Open() error {
continue
}
h.logger().Printf("opening database: %s", filepath.Base(fi.Name()))
h.logger().Printf("opening index: %s", filepath.Base(fi.Name()))
db, err := h.newDB(h.DBPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err == ErrName {
h.logger().Printf("ERROR opening database: %s, err=%s", fi.Name(), err)
h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err)
continue
} else if err != nil {
return err
}
if err := db.Open(); err != nil {
if err := index.Open(); err != nil {
if err == ErrName {
h.logger().Printf("ERROR opening database: %s, err=%s", db.Name(), err)
h.logger().Printf("ERROR opening index: %s, err=%s", index.Name(), err)
continue
}
return fmt.Errorf("open db: name=%s, err=%s", db.Name(), err)
return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err)
}
h.dbs[db.Name()] = db
h.indexes[index.Name()] = index
h.Stats.Count("dbN", 1)
h.Stats.Count("indexN", 1)
}
// Periodically flush cache.
@ -111,36 +111,36 @@ func (h *Holder) Close() error {
close(h.closing)
h.wg.Wait()
for _, db := range h.dbs {
db.Close()
for _, index := range h.indexes {
index.Close()
}
return nil
}
// MaxSlices returns MaxSlice map for all databases.
// MaxSlices returns MaxSlice map for all indexes.
func (h *Holder) MaxSlices() map[string]uint64 {
a := make(map[string]uint64)
for _, db := range h.DBs() {
a[db.Name()] = db.MaxSlice()
for _, index := range h.Indexes() {
a[index.Name()] = index.MaxSlice()
}
return a
}
// MaxInverseSlices returns MaxInverseSlice map for all databases.
// MaxInverseSlices returns MaxInverseSlice map for all indexes.
func (h *Holder) MaxInverseSlices() map[string]uint64 {
a := make(map[string]uint64)
for _, db := range h.DBs() {
a[db.Name()] = db.MaxInverseSlice()
for _, index := range h.Indexes() {
a[index.Name()] = index.MaxInverseSlice()
}
return a
}
// Schema returns schema data for all databases and frames.
func (h *Holder) Schema() []*DBInfo {
var a []*DBInfo
for _, db := range h.DBs() {
di := &DBInfo{Name: db.Name()}
for _, frame := range db.Frames() {
// Schema returns schema data for all indexes and frames.
func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
for _, index := range h.Indexes() {
di := &IndexInfo{Name: index.Name()}
for _, frame := range index.Frames() {
fi := &FrameInfo{Name: frame.Name()}
for _, view := range frame.Views() {
fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
@ -151,155 +151,155 @@ func (h *Holder) Schema() []*DBInfo {
sort.Sort(frameInfoSlice(di.Frames))
a = append(a, di)
}
sort.Sort(dbInfoSlice(a))
sort.Sort(indexInfoSlice(a))
return a
}
// DBPath returns the path where a given database is stored.
func (h *Holder) DBPath(name string) string { return filepath.Join(h.Path, name) }
// IndexPath returns the path where a given index is stored.
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
// DB returns the database by name.
func (h *Holder) DB(name string) *DB {
// Index returns the index by name.
func (h *Holder) Index(name string) *Index {
h.mu.Lock()
defer h.mu.Unlock()
return h.db(name)
return h.index(name)
}
func (h *Holder) db(name string) *DB { return h.dbs[name] }
func (h *Holder) index(name string) *Index { return h.indexes[name] }
// DBs returns a list of all databases in the holder.
func (h *Holder) DBs() []*DB {
// Indexes returns a list of all indexes in the holder.
func (h *Holder) Indexes() []*Index {
h.mu.Lock()
defer h.mu.Unlock()
a := make([]*DB, 0, len(h.dbs))
for _, db := range h.dbs {
a = append(a, db)
a := make([]*Index, 0, len(h.indexes))
for _, index := range h.indexes {
a = append(a, index)
}
sort.Sort(dbSlice(a))
sort.Sort(indexSlice(a))
return a
}
// CreateDB creates a database.
// An error is returned if the database already exists.
func (h *Holder) CreateDB(name string, opt DBOptions) (*DB, error) {
// CreateIndex creates an index.
// An error is returned if the index already exists.
func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
h.mu.Lock()
defer h.mu.Unlock()
// Ensure db doesn't already exist.
if h.dbs[name] != nil {
return nil, ErrDatabaseExists
// Ensure index doesn't already exist.
if h.indexes[name] != nil {
return nil, ErrIndexExists
}
return h.createDB(name, opt)
return h.createIndex(name, opt)
}
// CreateDBIfNotExists returns a database by name.
// The database is created if it does not already exist.
func (h *Holder) CreateDBIfNotExists(name string, opt DBOptions) (*DB, error) {
// CreateIndexIfNotExists returns an index by name.
// The index is created if it does not already exist.
func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index, error) {
h.mu.Lock()
defer h.mu.Unlock()
// Find database in cache first.
if db := h.dbs[name]; db != nil {
return db, nil
// Find index in cache first.
if index := h.indexes[name]; index != nil {
return index, nil
}
return h.createDB(name, opt)
return h.createIndex(name, opt)
}
func (h *Holder) createDB(name string, opt DBOptions) (*DB, error) {
func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
if name == "" {
return nil, errors.New("database name required")
return nil, errors.New("index name required")
}
// Return database if it exists.
if db := h.db(name); db != nil {
return db, nil
// Return index if it exists.
if index := h.index(name); index != nil {
return index, nil
}
// Otherwise create a new database.
db, err := h.newDB(h.DBPath(name), name)
// Otherwise create a new index.
index, err := h.newIndex(h.IndexPath(name), name)
if err != nil {
return nil, err
}
if err := db.Open(); err != nil {
if err := index.Open(); err != nil {
return nil, err
}
// Update options.
db.SetColumnLabel(opt.ColumnLabel)
db.SetTimeQuantum(opt.TimeQuantum)
index.SetColumnLabel(opt.ColumnLabel)
index.SetTimeQuantum(opt.TimeQuantum)
h.dbs[db.Name()] = db
h.indexes[index.Name()] = index
h.Stats.Count("dbN", 1)
h.Stats.Count("indexN", 1)
return db, nil
return index, nil
}
func (h *Holder) newDB(path, name string) (*DB, error) {
db, err := NewDB(path, name)
func (h *Holder) newIndex(path, name string) (*Index, error) {
index, err := NewIndex(path, name)
if err != nil {
return nil, err
}
db.LogOutput = h.LogOutput
db.stats = h.Stats.WithTags(fmt.Sprintf("db:%s", db.Name()))
db.broadcaster = h.Broadcaster
return db, nil
index.LogOutput = h.LogOutput
index.stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
index.broadcaster = h.Broadcaster
return index, nil
}
// DeleteDB removes a database from the holder.
func (h *Holder) DeleteDB(name string) error {
// DeleteIndex removes an index from the holder.
func (h *Holder) DeleteIndex(name string) error {
h.mu.Lock()
defer h.mu.Unlock()
// Ignore if database doesn't exist.
db := h.db(name)
if db == nil {
// Ignore if index doesn't exist.
index := h.index(name)
if index == nil {
return nil
}
// Close database.
if err := db.Close(); err != nil {
// Close index.
if err := index.Close(); err != nil {
return err
}
// Delete database directory.
if err := os.RemoveAll(h.DBPath(name)); err != nil {
// Delete index directory.
if err := os.RemoveAll(h.IndexPath(name)); err != nil {
return err
}
// Remove reference.
delete(h.dbs, name)
delete(h.indexes, name)
h.Stats.Count("dbN", -1)
h.Stats.Count("indexN", -1)
return nil
}
// Frame returns the frame for a database and name.
func (h *Holder) Frame(db, name string) *Frame {
d := h.DB(db)
// Frame returns the frame for an index and name.
func (h *Holder) Frame(index, name string) *Frame {
d := h.Index(index)
if d == nil {
return nil
}
return d.Frame(name)
}
// View returns the view for a database, frame, and name.
func (h *Holder) View(db, frame, name string) *View {
f := h.Frame(db, frame)
// View returns the view for an index, frame, and name.
func (h *Holder) View(index, frame, name string) *View {
f := h.Frame(index, frame)
if f == nil {
return nil
}
return f.View(name)
}
// Fragment returns the fragment for a database, frame & slice.
func (h *Holder) Fragment(db, frame, view string, slice uint64) *Fragment {
v := h.View(db, frame, view)
// Fragment returns the fragment for an index, frame & slice.
func (h *Holder) Fragment(index, frame, view string, slice uint64) *Fragment {
v := h.View(index, frame, view)
if v == nil {
return nil
}
@ -323,8 +323,8 @@ func (h *Holder) monitorCacheFlush() {
}
func (h *Holder) flushCaches() {
for _, db := range h.DBs() {
for _, frame := range db.Frames() {
for _, index := range h.Indexes() {
for _, frame := range index.Frames() {
for _, view := range frame.Views() {
for _, fragment := range view.Fragments() {
select {
@ -375,9 +375,9 @@ func (s *HolderSyncer) SyncHolder() error {
return nil
}
// Sync database column attributes.
if err := s.syncDatabase(di.Name); err != nil {
return fmt.Errorf("db sync error: db=%s, err=%s", di.Name, err)
// Sync index column attributes.
if err := s.syncIndex(di.Name); err != nil {
return fmt.Errorf("index sync error: index=%s, err=%s", di.Name, err)
}
for _, fi := range di.Frames {
@ -388,7 +388,7 @@ func (s *HolderSyncer) SyncHolder() error {
// Sync frame row attributes.
if err := s.syncFrame(di.Name, fi.Name); err != nil {
return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err)
return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err)
}
for _, vi := range fi.Views {
@ -397,7 +397,7 @@ func (s *HolderSyncer) SyncHolder() error {
return nil
}
for slice := uint64(0); slice <= s.Holder.DB(di.Name).MaxSlice(); slice++ {
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
// Ignore slices that this host doesn't own.
if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) {
continue
@ -410,7 +410,7 @@ func (s *HolderSyncer) SyncHolder() error {
// Sync fragment if own it.
if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil {
return fmt.Errorf("fragment sync error: db=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
return fmt.Errorf("fragment sync error: index=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
}
}
}
@ -420,10 +420,10 @@ func (s *HolderSyncer) SyncHolder() error {
return nil
}
// syncDatabase synchronizes database attributes with the rest of the cluster.
func (s *HolderSyncer) syncDatabase(db string) error {
// Retrieve database reference.
d := s.Holder.DB(db)
// syncIndex synchronizes index attributes with the rest of the cluster.
func (s *HolderSyncer) syncIndex(index string) error {
// Retrieve index reference.
d := s.Holder.Index(index)
if d == nil {
return nil
}
@ -443,7 +443,7 @@ func (s *HolderSyncer) syncDatabase(db string) error {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := client.ColumnAttrDiff(context.Background(), db, blks)
m, err := client.ColumnAttrDiff(context.Background(), index, blks)
if err != nil {
return err
} else if len(m) == 0 {
@ -466,9 +466,9 @@ func (s *HolderSyncer) syncDatabase(db string) error {
}
// syncFrame synchronizes frame attributes with the rest of the cluster.
func (s *HolderSyncer) syncFrame(db, name string) error {
// Retrieve database reference.
f := s.Holder.Frame(db, name)
func (s *HolderSyncer) syncFrame(index, name string) error {
// Retrieve index reference.
f := s.Holder.Frame(index, name)
if f == nil {
return nil
}
@ -488,7 +488,7 @@ func (s *HolderSyncer) syncFrame(db, name string) error {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := client.RowAttrDiff(context.Background(), db, name, blks)
m, err := client.RowAttrDiff(context.Background(), index, name, blks)
if err == ErrFrameNotFound {
continue // frame not created remotely yet, skip
} else if err != nil {
@ -513,9 +513,9 @@ func (s *HolderSyncer) syncFrame(db, name string) error {
}
// syncFragment synchronizes a fragment with the rest of the cluster.
func (s *HolderSyncer) syncFragment(db, frame, view string, slice uint64) error {
func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error {
// Retrieve local frame.
f := s.Holder.Frame(db, frame)
f := s.Holder.Frame(index, frame)
if f == nil {
return ErrFrameNotFound
}

View file

@ -12,36 +12,36 @@ import (
"github.com/pilosa/pilosa/pql"
)
// Ensure holder can delete a database and its underlying files.
func TestHolder_DeleteDB(t *testing.T) {
// Ensure holder can delete an index and its underlying files.
func TestHolder_DeleteIndex(t *testing.T) {
hldr := MustOpenHolder()
defer hldr.Close()
// Write bits to separate databases.
f0 := hldr.MustCreateFragmentIfNotExists("d0", "f", pilosa.ViewStandard, 0)
// Write bits to separate indexes.
f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0)
if _, err := f0.SetBit(100, 200); err != nil {
t.Fatal(err)
}
f1 := hldr.MustCreateFragmentIfNotExists("d1", "f", pilosa.ViewStandard, 0)
f1 := hldr.MustCreateFragmentIfNotExists("i1", "f", pilosa.ViewStandard, 0)
if _, err := f1.SetBit(100, 200); err != nil {
t.Fatal(err)
}
// Ensure d0 exists.
if _, err := os.Stat(hldr.DBPath("d0")); err != nil {
// Ensure i0 exists.
if _, err := os.Stat(hldr.IndexPath("i0")); err != nil {
t.Fatal(err)
}
// Delete d0.
if err := hldr.DeleteDB("d0"); err != nil {
// Delete i0.
if err := hldr.DeleteIndex("i0"); err != nil {
t.Fatal(err)
}
// Ensure d0 files are removed & d1 still exists.
if _, err := os.Stat(hldr.DBPath("d0")); !os.IsNotExist(err) {
t.Fatal("expected d0 file deletion")
} else if _, err := os.Stat(hldr.DBPath("d1")); err != nil {
t.Fatal("expected d1 files to still exist", err)
// Ensure i0 files are removed & i1 still exists.
if _, err := os.Stat(hldr.IndexPath("i0")); !os.IsNotExist(err) {
t.Fatal("expected i0 file deletion")
} else if _, err := os.Stat(hldr.IndexPath("i1")); err != nil {
t.Fatal("expected i1 files to still exist", err)
}
}
@ -59,12 +59,12 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
s := NewServer()
defer s.Close()
s.Handler.Holder = hldr1.Holder
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
e := pilosa.NewExecutor()
e.Holder = hldr1.Holder
e.Host = cluster.Nodes[1].Host
e.Cluster = cluster
return e.Execute(ctx, db, query, slices, opt)
return e.Execute(ctx, index, query, slices, opt)
}
// Mock 2-node, fully replicated cluster.
@ -74,13 +74,13 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Create frames on nodes.
for _, hldr := range []*Holder{hldr0, hldr1} {
hldr.MustCreateFrameIfNotExists("d", "f")
hldr.MustCreateFrameIfNotExists("d", "f0")
hldr.MustCreateFrameIfNotExists("i", "f")
hldr.MustCreateFrameIfNotExists("i", "f0")
hldr.MustCreateFrameIfNotExists("y", "z")
}
// Set data on the local holder.
f := hldr0.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 10); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 20); err != nil {
@ -91,7 +91,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
t.Fatal(err)
}
f = hldr0.MustCreateFragmentIfNotExists("d", "f0", pilosa.ViewStandard, 1)
f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
t.Fatal(err)
}
@ -99,7 +99,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0)
// Set data on the remote holder.
f = hldr1.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
f = hldr1.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if _, err := f.SetBit(0, 4000); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(3, 10); err != nil {
@ -118,8 +118,8 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
}
// Set highest slice.
hldr0.DB("d").SetRemoteMaxSlice(1)
hldr0.DB("y").SetRemoteMaxSlice(3)
hldr0.Index("i").SetRemoteMaxSlice(1)
hldr0.Index("y").SetRemoteMaxSlice(3)
// Set up syncer.
syncer := pilosa.HolderSyncer{
@ -134,7 +134,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*Holder{hldr0, hldr1} {
f := hldr.Fragment("d", "f", pilosa.ViewStandard, 0)
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
@ -147,7 +147,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
}
f = hldr.Fragment("d", "f0", pilosa.ViewStandard, 1)
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
@ -197,18 +197,18 @@ func (h *Holder) Close() error {
return h.Holder.Close()
}
// MustCreateDBIfNotExists returns a given db. Panic on error.
func (h *Holder) MustCreateDBIfNotExists(db string, opt pilosa.DBOptions) *DB {
d, err := h.Holder.CreateDBIfNotExists(db, opt)
// MustCreateIndexIfNotExists returns a given index. Panic on error.
func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index {
d, err := h.Holder.CreateIndexIfNotExists(index, opt)
if err != nil {
panic(err)
}
return &DB{DB: d}
return &Index{Index: d}
}
// MustCreateFrameIfNotExists returns a given frame. Panic on error.
func (h *Holder) MustCreateFrameIfNotExists(db, frame string) *Frame {
f, err := h.MustCreateDBIfNotExists(db, pilosa.DBOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame {
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
if err != nil {
panic(err)
}
@ -216,8 +216,8 @@ func (h *Holder) MustCreateFrameIfNotExists(db, frame string) *Frame {
}
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
func (h *Holder) MustCreateFragmentIfNotExists(db, frame, view string, slice uint64) *Fragment {
d := h.MustCreateDBIfNotExists(db, pilosa.DBOptions{})
func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
d := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
if err != nil {
panic(err)

565
index.go Normal file
View file

@ -0,0 +1,565 @@
package pilosa
import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"sort"
"sync"
"time"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa/internal"
)
// Default index settings.
const (
DefaultColumnLabel = "columnID"
)
// Index represents a container for frames.
type Index struct {
mu sync.Mutex
path string
name string
// Default time quantum for all frames in index.
// This can be overridden by individual frames.
timeQuantum TimeQuantum
// Label used for referring to columns in index.
columnLabel string
// Frames by name.
frames map[string]*Frame
// Max Slice on any node in the cluster, according to this node
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
// Column attribute storage and cache
columnAttrStore *AttrStore
broadcaster Broadcaster
stats StatsClient
LogOutput io.Writer
}
// NewIndex returns a new instance of Index.
func NewIndex(path, name string) (*Index, error) {
err := ValidateName(name)
if err != nil {
return nil, err
}
return &Index{
path: path,
name: name,
frames: make(map[string]*Frame),
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
columnAttrStore: NewAttrStore(filepath.Join(path, ".data")),
columnLabel: DefaultColumnLabel,
stats: NopStatsClient,
LogOutput: ioutil.Discard,
}, nil
}
// Name returns name of the index.
func (i *Index) Name() string { return i.name }
// Path returns the path the index was initialized with.
func (i *Index) Path() string { return i.path }
// ColumnAttrStore returns the storage for column attributes.
func (i *Index) ColumnAttrStore() *AttrStore { return i.columnAttrStore }
// SetColumnLabel sets the column label. Persists to meta file on update.
func (i *Index) SetColumnLabel(v string) error {
i.mu.Lock()
defer i.mu.Unlock()
// Ignore if no change occurred.
if v == "" || i.columnLabel == v {
return nil
}
// Make sure columnLabel is valid name
err := ValidateName(v)
if err != nil {
return err
}
// Persist meta data to disk on change.
i.columnLabel = v
if err := i.saveMeta(); err != nil {
return err
}
return nil
}
// ColumnLabel returns the column label.
func (i *Index) ColumnLabel() string {
i.mu.Lock()
v := i.columnLabel
i.mu.Unlock()
return v
}
// Open opens and initializes the index.
func (i *Index) Open() error {
// Ensure the path exists.
if err := os.MkdirAll(i.path, 0777); err != nil {
return err
}
// Read meta file.
if err := i.loadMeta(); err != nil {
return err
}
if err := i.openFrames(); err != nil {
return err
}
if err := i.columnAttrStore.Open(); err != nil {
return err
}
return nil
}
// openFrames opens and initializes the frames inside the index.
func (i *Index) openFrames() error {
f, err := os.Open(i.path)
if err != nil {
return err
}
defer f.Close()
fis, err := f.Readdir(0)
if err != nil {
return err
}
for _, fi := range fis {
if !fi.IsDir() {
continue
}
fr, err := i.newFrame(i.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
if err != nil {
return ErrName
}
if err := fr.Open(); err != nil {
return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err)
}
i.frames[fr.Name()] = fr
i.stats.Count("frameN", 1)
}
return nil
}
// loadMeta reads meta data for the index, if any.
func (i *Index) loadMeta() error {
var pb internal.IndexMeta
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(i.path, ".meta"))
if os.IsNotExist(err) {
i.timeQuantum = ""
i.columnLabel = DefaultColumnLabel
return nil
} else if err != nil {
return err
} else {
if err := proto.Unmarshal(buf, &pb); err != nil {
return err
}
}
// Copy metadata fields.
i.timeQuantum = TimeQuantum(pb.TimeQuantum)
i.columnLabel = pb.ColumnLabel
return nil
}
// saveMeta writes meta data for the index.
func (i *Index) saveMeta() error {
// Marshal metadata.
buf, err := proto.Marshal(&internal.IndexMeta{
TimeQuantum: string(i.timeQuantum),
ColumnLabel: i.columnLabel,
})
if err != nil {
return err
}
// Write to meta file.
if err := ioutil.WriteFile(filepath.Join(i.path, ".meta"), buf, 0666); err != nil {
return err
}
return nil
}
// Close closes the index and its frames.
func (i *Index) Close() error {
i.mu.Lock()
defer i.mu.Unlock()
// Close the attribute store.
if i.columnAttrStore != nil {
i.columnAttrStore.Close()
}
// Close all frames.
for _, f := range i.frames {
f.Close()
}
i.frames = make(map[string]*Frame)
return nil
}
// MaxSlice returns the max slice in the index according to this node.
func (i *Index) MaxSlice() uint64 {
if i == nil {
return 0
}
i.mu.Lock()
defer i.mu.Unlock()
max := i.remoteMaxSlice
for _, f := range i.frames {
if slice := f.MaxSlice(); slice > max {
max = slice
}
}
return max
}
func (i *Index) SetRemoteMaxSlice(newmax uint64) {
i.mu.Lock()
defer i.mu.Unlock()
i.remoteMaxSlice = newmax
}
// MaxInverseSlice returns the max inverse slice in the index according to this node.
func (i *Index) MaxInverseSlice() uint64 {
if i == nil {
return 0
}
i.mu.Lock()
defer i.mu.Unlock()
max := i.remoteMaxInverseSlice
for _, f := range i.frames {
if slice := f.MaxInverseSlice(); slice > max {
max = slice
}
}
return max
}
func (i *Index) SetRemoteMaxInverseSlice(v uint64) {
i.mu.Lock()
defer i.mu.Unlock()
i.remoteMaxInverseSlice = v
}
// TimeQuantum returns the default time quantum for the index.
func (i *Index) TimeQuantum() TimeQuantum {
i.mu.Lock()
defer i.mu.Unlock()
return i.timeQuantum
}
// SetTimeQuantum sets the default time quantum for the index.
func (i *Index) SetTimeQuantum(q TimeQuantum) error {
i.mu.Lock()
defer i.mu.Unlock()
// Validate input.
if !q.Valid() {
return ErrInvalidTimeQuantum
}
// Update value on index.
i.timeQuantum = q
// Perist meta data to disk.
if err := i.saveMeta(); err != nil {
return err
}
return nil
}
// FramePath returns the path to a frame in the index.
func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
// Frame returns a frame in the index by name.
func (i *Index) Frame(name string) *Frame {
i.mu.Lock()
defer i.mu.Unlock()
return i.frame(name)
}
func (i *Index) frame(name string) *Frame { return i.frames[name] }
// Frames returns a list of all frames in the index.
func (i *Index) Frames() []*Frame {
i.mu.Lock()
defer i.mu.Unlock()
a := make([]*Frame, 0, len(i.frames))
for _, f := range i.frames {
a = append(a, f)
}
sort.Sort(frameSlice(a))
return a
}
// CreateFrame creates a frame.
func (i *Index) CreateFrame(name string, opt FrameOptions) (*Frame, error) {
i.mu.Lock()
defer i.mu.Unlock()
// Ensure frame doesn't already exist.
if i.frames[name] != nil {
return nil, ErrFrameExists
}
return i.createFrame(name, opt)
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (i *Index) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) {
i.mu.Lock()
defer i.mu.Unlock()
// Find frame in cache first.
if f := i.frames[name]; f != nil {
return f, nil
}
return i.createFrame(name, opt)
}
func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
if name == "" {
return nil, errors.New("frame name required")
} else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) {
return nil, ErrInvalidCacheType
}
// Initialize frame.
f, err := i.newFrame(i.FramePath(name), name)
if err != nil {
return nil, err
}
// Open frame.
if err := f.Open(); err != nil {
return nil, err
}
// Default the time quantum to what is set on the Index.
if err := f.SetTimeQuantum(i.timeQuantum); err != nil {
f.Close()
return nil, err
}
// Set cache type.
if opt.CacheType == "" {
opt.CacheType = DefaultCacheType
}
f.cacheType = opt.CacheType
// Set options.
if opt.RowLabel != "" {
f.rowLabel = opt.RowLabel
}
if opt.CacheSize != 0 {
f.cacheSize = opt.CacheSize
}
f.inverseEnabled = opt.InverseEnabled
if err := f.saveMeta(); err != nil {
f.Close()
return nil, err
}
// Add to index's frame lookup.
i.frames[name] = f
i.stats.Count("frameN", 1)
return f, nil
}
func (i *Index) newFrame(path, name string) (*Frame, error) {
f, err := NewFrame(path, i.name, name)
if err != nil {
return nil, err
}
f.LogOutput = i.LogOutput
f.stats = i.stats.WithTags(fmt.Sprintf("frame:%s", name))
f.broadcaster = i.broadcaster
return f, nil
}
// DeleteFrame removes a frame from the index.
func (i *Index) DeleteFrame(name string) error {
i.mu.Lock()
defer i.mu.Unlock()
// Ignore if frame doesn't exist.
f := i.frame(name)
if f == nil {
return nil
}
// Close frame.
if err := f.Close(); err != nil {
return err
}
// Delete frame directory.
if err := os.RemoveAll(i.FramePath(name)); err != nil {
return err
}
// Remove reference.
delete(i.frames, name)
i.stats.Count("frameN", -1)
return nil
}
type indexSlice []*Index
func (p indexSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p indexSlice) Len() int { return len(p) }
func (p indexSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
// IndexInfo represents schema information for an index.
type IndexInfo struct {
Name string `json:"name"`
Frames []*FrameInfo `json:"frames"`
}
type indexInfoSlice []*IndexInfo
func (p indexInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p indexInfoSlice) Len() int { return len(p) }
func (p indexInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// MergeSchemas combines indexes and frames from a and b into one schema.
func MergeSchemas(a, b []*IndexInfo) []*IndexInfo {
// Generate a map from both schemas.
m := make(map[string]map[string]map[string]struct{})
for _, idxs := range [][]*IndexInfo{a, b} {
for _, idx := range idxs {
if m[idx.Name] == nil {
m[idx.Name] = make(map[string]map[string]struct{})
}
for _, frame := range idx.Frames {
if m[idx.Name][frame.Name] == nil {
m[idx.Name][frame.Name] = make(map[string]struct{})
}
for _, view := range frame.Views {
m[idx.Name][frame.Name][view.Name] = struct{}{}
}
}
}
}
// Generate new schema from map.
idxs := make([]*IndexInfo, 0, len(m))
for idx, frames := range m {
di := &IndexInfo{Name: idx}
for frame, views := range frames {
fi := &FrameInfo{Name: frame}
for view := range views {
fi.Views = append(fi.Views, &ViewInfo{Name: view})
}
sort.Sort(viewInfoSlice(fi.Views))
di.Frames = append(di.Frames, fi)
}
sort.Sort(frameInfoSlice(di.Frames))
idxs = append(idxs, di)
}
sort.Sort(indexInfoSlice(idxs))
return idxs
}
// encodeIndexes converts a into its internal representation.
func encodeIndexes(a []*Index) []*internal.Index {
other := make([]*internal.Index, len(a))
for i := range a {
other[i] = encodeIndex(a[i])
}
return other
}
// encodeIndex converts d into its internal representation.
func encodeIndex(d *Index) *internal.Index {
return &internal.Index{
Name: d.name,
Meta: &internal.IndexMeta{
ColumnLabel: d.columnLabel,
TimeQuantum: string(d.timeQuantum),
},
MaxSlice: d.remoteMaxSlice,
Frames: encodeFrames(d.Frames()),
}
}
// IndexOptions represents options to set when initializing an index.
type IndexOptions struct {
ColumnLabel string `json:"columnLabel,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
}
// Encode converts o into its internal representation.
func (o *IndexOptions) Encode() *internal.IndexMeta {
return &internal.IndexMeta{
ColumnLabel: o.ColumnLabel,
TimeQuantum: string(o.TimeQuantum),
}
}
// hasTime returns true if a contains a non-nil time.
func hasTime(a []*time.Time) bool {
for _, t := range a {
if t != nil {
return true
}
}
return false
}
type importKey struct {
View string
Slice uint64
}
type importData struct {
RowIDs []uint64
ColumnIDs []uint64
}

179
index_test.go Normal file
View file

@ -0,0 +1,179 @@
package pilosa_test
import (
"io/ioutil"
"os"
"testing"
"github.com/pilosa/pilosa"
)
// Ensure index can open and retrieve a frame.
func TestIndex_CreateFrameIfNotExists(t *testing.T) {
index := MustOpenIndex()
defer index.Close()
// Create frame.
f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if f == nil {
t.Fatal("expected frame")
}
// Retrieve existing frame.
other, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if f.Frame != other.Frame {
t.Fatal("frame mismatch")
}
if f.Frame != index.Frame("f") {
t.Fatal("frame mismatch")
}
}
// Ensure index defaults the time quantum on new frames.
func TestIndex_CreateFrame_TimeQuantum(t *testing.T) {
index := MustOpenIndex()
defer index.Close()
// Set index time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
t.Fatal(err)
}
// Create frame.
f, err := index.CreateFrame("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") {
t.Fatalf("unexpected frame time quantum: %s", q)
}
}
// Ensure index can delete a frame.
func TestIndex_DeleteFrame(t *testing.T) {
index := MustOpenIndex()
defer index.Close()
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Delete frame & verify it's gone.
if err := index.DeleteFrame("f"); err != nil {
t.Fatal(err)
} else if index.Frame("f") != nil {
t.Fatal("expected nil frame")
}
// Delete again to make sure it doesn't error.
if err := index.DeleteFrame("f"); err != nil {
t.Fatal(err)
}
}
// Ensure index can set the default time quantum.
func TestIndex_SetTimeQuantum(t *testing.T) {
index := MustOpenIndex()
defer index.Close()
// Set & retrieve time quantum.
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
t.Fatal(err)
} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum: %s", q)
}
// Reload index and verify that it is persisted.
if err := index.Reopen(); err != nil {
t.Fatal(err)
} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// Index represents a test wrapper for pilosa.Index.
type Index struct {
*pilosa.Index
}
// NewIndex returns a new instance of Index.
func NewIndex() *Index {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := pilosa.NewIndex(path, "i")
if err != nil {
panic(err)
}
return &Index{Index: index}
}
// MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func MustOpenIndex() *Index {
index := NewIndex()
if err := index.Open(); err != nil {
panic(err)
}
return index
}
// Close closes the index and removes the underlying data.
func (i *Index) Close() error {
defer os.RemoveAll(i.Path())
return i.Index.Close()
}
// Reopen closes the index and reopens it.
func (i *Index) Reopen() error {
var err error
if err := i.Index.Close(); err != nil {
return err
}
path, name := i.Path(), i.Name()
i.Index, err = pilosa.NewIndex(path, name)
if err != nil {
return err
}
if err := i.Open(); err != nil {
return err
}
return nil
}
// CreateFrame creates a frame with the given options.
func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrame(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrameIfNotExists(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
}
// Ensure index can delete a frame.
func TestIndex_InvalidName(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := pilosa.NewIndex(path, "ABC")
if index != nil {
t.Fatalf("unexpected index name %s", index)
}
}

View file

@ -9,7 +9,7 @@
private.proto
It has these top-level messages:
DBMeta
IndexMeta
FrameMeta
ImportResponse
BlockDataRequest
@ -17,12 +17,12 @@
Cache
MaxSlicesResponse
CreateSliceMessage
DeleteDBMessage
CreateDBMessage
DeleteIndexMessage
CreateIndexMessage
CreateFrameMessage
DeleteFrameMessage
Frame
DB
Index
NodeState
*/
package internal
@ -44,15 +44,15 @@ var _ = math.Inf
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type DBMeta struct {
type IndexMeta struct {
ColumnLabel string `protobuf:"bytes,1,opt,name=ColumnLabel,proto3" json:"ColumnLabel,omitempty"`
TimeQuantum string `protobuf:"bytes,2,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
}
func (m *DBMeta) Reset() { *m = DBMeta{} }
func (m *DBMeta) String() string { return proto.CompactTextString(m) }
func (*DBMeta) ProtoMessage() {}
func (*DBMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} }
func (m *IndexMeta) Reset() { *m = IndexMeta{} }
func (m *IndexMeta) String() string { return proto.CompactTextString(m) }
func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} }
type FrameMeta struct {
RowLabel string `protobuf:"bytes,1,opt,name=RowLabel,proto3" json:"RowLabel,omitempty"`
@ -77,7 +77,7 @@ func (*ImportResponse) ProtoMessage() {}
func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{2} }
type BlockDataRequest struct {
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
View string `protobuf:"bytes,5,opt,name=View,proto3" json:"View,omitempty"`
Slice uint64 `protobuf:"varint,4,opt,name=Slice,proto3" json:"Slice,omitempty"`
@ -125,7 +125,7 @@ func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 {
}
type CreateSliceMessage struct {
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
}
@ -134,26 +134,26 @@ func (m *CreateSliceMessage) String() string { return proto.CompactTe
func (*CreateSliceMessage) ProtoMessage() {}
func (*CreateSliceMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{7} }
type DeleteDBMessage struct {
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
type DeleteIndexMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
}
func (m *DeleteDBMessage) Reset() { *m = DeleteDBMessage{} }
func (m *DeleteDBMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteDBMessage) ProtoMessage() {}
func (*DeleteDBMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} }
func (m *DeleteIndexMessage) Reset() { *m = DeleteIndexMessage{} }
func (m *DeleteIndexMessage) String() string { return proto.CompactTextString(m) }
func (*DeleteIndexMessage) ProtoMessage() {}
func (*DeleteIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{8} }
type CreateDBMessage struct {
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
Meta *DBMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"`
type CreateIndexMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"`
}
func (m *CreateDBMessage) Reset() { *m = CreateDBMessage{} }
func (m *CreateDBMessage) String() string { return proto.CompactTextString(m) }
func (*CreateDBMessage) ProtoMessage() {}
func (*CreateDBMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} }
func (m *CreateIndexMessage) Reset() { *m = CreateIndexMessage{} }
func (m *CreateIndexMessage) String() string { return proto.CompactTextString(m) }
func (*CreateIndexMessage) ProtoMessage() {}
func (*CreateIndexMessage) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{9} }
func (m *CreateDBMessage) GetMeta() *DBMeta {
func (m *CreateIndexMessage) GetMeta() *IndexMeta {
if m != nil {
return m.Meta
}
@ -161,7 +161,7 @@ func (m *CreateDBMessage) GetMeta() *DBMeta {
}
type CreateFrameMessage struct {
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
Meta *FrameMeta `protobuf:"bytes,3,opt,name=Meta" json:"Meta,omitempty"`
}
@ -179,7 +179,7 @@ func (m *CreateFrameMessage) GetMeta() *FrameMeta {
}
type DeleteFrameMessage struct {
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
}
@ -205,26 +205,26 @@ func (m *Frame) GetMeta() *FrameMeta {
return nil
}
type DB struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Meta *DBMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"`
MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"`
Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"`
type Index struct {
Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"`
Meta *IndexMeta `protobuf:"bytes,2,opt,name=Meta" json:"Meta,omitempty"`
MaxSlice uint64 `protobuf:"varint,3,opt,name=MaxSlice,proto3" json:"MaxSlice,omitempty"`
Frames []*Frame `protobuf:"bytes,4,rep,name=Frames" json:"Frames,omitempty"`
}
func (m *DB) Reset() { *m = DB{} }
func (m *DB) String() string { return proto.CompactTextString(m) }
func (*DB) ProtoMessage() {}
func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} }
func (m *Index) Reset() { *m = Index{} }
func (m *Index) String() string { return proto.CompactTextString(m) }
func (*Index) ProtoMessage() {}
func (*Index) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{13} }
func (m *DB) GetMeta() *DBMeta {
func (m *Index) GetMeta() *IndexMeta {
if m != nil {
return m.Meta
}
return nil
}
func (m *DB) GetFrames() []*Frame {
func (m *Index) GetFrames() []*Frame {
if m != nil {
return m.Frames
}
@ -232,9 +232,9 @@ func (m *DB) GetFrames() []*Frame {
}
type NodeState struct {
Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"`
State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
DBs []*DB `protobuf:"bytes,3,rep,name=DBs" json:"DBs,omitempty"`
Host string `protobuf:"bytes,1,opt,name=Host,proto3" json:"Host,omitempty"`
State string `protobuf:"bytes,2,opt,name=State,proto3" json:"State,omitempty"`
Indexes []*Index `protobuf:"bytes,3,rep,name=Indexes" json:"Indexes,omitempty"`
}
func (m *NodeState) Reset() { *m = NodeState{} }
@ -242,15 +242,15 @@ func (m *NodeState) String() string { return proto.CompactTextString(
func (*NodeState) ProtoMessage() {}
func (*NodeState) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{14} }
func (m *NodeState) GetDBs() []*DB {
func (m *NodeState) GetIndexes() []*Index {
if m != nil {
return m.DBs
return m.Indexes
}
return nil
}
func init() {
proto.RegisterType((*DBMeta)(nil), "internal.DBMeta")
proto.RegisterType((*IndexMeta)(nil), "internal.IndexMeta")
proto.RegisterType((*FrameMeta)(nil), "internal.FrameMeta")
proto.RegisterType((*ImportResponse)(nil), "internal.ImportResponse")
proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest")
@ -258,15 +258,15 @@ func init() {
proto.RegisterType((*Cache)(nil), "internal.Cache")
proto.RegisterType((*MaxSlicesResponse)(nil), "internal.MaxSlicesResponse")
proto.RegisterType((*CreateSliceMessage)(nil), "internal.CreateSliceMessage")
proto.RegisterType((*DeleteDBMessage)(nil), "internal.DeleteDBMessage")
proto.RegisterType((*CreateDBMessage)(nil), "internal.CreateDBMessage")
proto.RegisterType((*DeleteIndexMessage)(nil), "internal.DeleteIndexMessage")
proto.RegisterType((*CreateIndexMessage)(nil), "internal.CreateIndexMessage")
proto.RegisterType((*CreateFrameMessage)(nil), "internal.CreateFrameMessage")
proto.RegisterType((*DeleteFrameMessage)(nil), "internal.DeleteFrameMessage")
proto.RegisterType((*Frame)(nil), "internal.Frame")
proto.RegisterType((*DB)(nil), "internal.DB")
proto.RegisterType((*Index)(nil), "internal.Index")
proto.RegisterType((*NodeState)(nil), "internal.NodeState")
}
func (m *DBMeta) Marshal() (dAtA []byte, err error) {
func (m *IndexMeta) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -276,7 +276,7 @@ func (m *DBMeta) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *DBMeta) MarshalTo(dAtA []byte) (int, error) {
func (m *IndexMeta) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
@ -386,11 +386,11 @@ func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.DB) > 0 {
if len(m.Index) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB)))
i += copy(dAtA[i:], m.DB)
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Frame) > 0 {
dAtA[i] = 0x12
@ -553,11 +553,11 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.DB) > 0 {
if len(m.Index) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB)))
i += copy(dAtA[i:], m.DB)
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if m.Slice != 0 {
dAtA[i] = 0x10
@ -567,7 +567,7 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *DeleteDBMessage) Marshal() (dAtA []byte, err error) {
func (m *DeleteIndexMessage) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -577,21 +577,21 @@ func (m *DeleteDBMessage) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *DeleteDBMessage) MarshalTo(dAtA []byte) (int, error) {
func (m *DeleteIndexMessage) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.DB) > 0 {
if len(m.Index) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB)))
i += copy(dAtA[i:], m.DB)
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
return i, nil
}
func (m *CreateDBMessage) Marshal() (dAtA []byte, err error) {
func (m *CreateIndexMessage) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -601,16 +601,16 @@ func (m *CreateDBMessage) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *CreateDBMessage) MarshalTo(dAtA []byte) (int, error) {
func (m *CreateIndexMessage) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if len(m.DB) > 0 {
if len(m.Index) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB)))
i += copy(dAtA[i:], m.DB)
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if m.Meta != nil {
dAtA[i] = 0x12
@ -640,11 +640,11 @@ func (m *CreateFrameMessage) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.DB) > 0 {
if len(m.Index) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB)))
i += copy(dAtA[i:], m.DB)
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Frame) > 0 {
dAtA[i] = 0x12
@ -680,11 +680,11 @@ func (m *DeleteFrameMessage) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.DB) > 0 {
if len(m.Index) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(m.DB)))
i += copy(dAtA[i:], m.DB)
i = encodeVarintPrivate(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Frame) > 0 {
dAtA[i] = 0x12
@ -729,7 +729,7 @@ func (m *Frame) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *DB) Marshal() (dAtA []byte, err error) {
func (m *Index) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -739,7 +739,7 @@ func (m *DB) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *DB) MarshalTo(dAtA []byte) (int, error) {
func (m *Index) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
@ -807,8 +807,8 @@ func (m *NodeState) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPrivate(dAtA, i, uint64(len(m.State)))
i += copy(dAtA[i:], m.State)
}
if len(m.DBs) > 0 {
for _, msg := range m.DBs {
if len(m.Indexes) > 0 {
for _, msg := range m.Indexes {
dAtA[i] = 0x1a
i++
i = encodeVarintPrivate(dAtA, i, uint64(msg.Size()))
@ -849,7 +849,7 @@ func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int {
dAtA[offset] = uint8(v)
return offset + 1
}
func (m *DBMeta) Size() (n int) {
func (m *IndexMeta) Size() (n int) {
var l int
_ = l
l = len(m.ColumnLabel)
@ -900,7 +900,7 @@ func (m *ImportResponse) Size() (n int) {
func (m *BlockDataRequest) Size() (n int) {
var l int
_ = l
l = len(m.DB)
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
@ -971,7 +971,7 @@ func (m *MaxSlicesResponse) Size() (n int) {
func (m *CreateSliceMessage) Size() (n int) {
var l int
_ = l
l = len(m.DB)
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
@ -981,20 +981,20 @@ func (m *CreateSliceMessage) Size() (n int) {
return n
}
func (m *DeleteDBMessage) Size() (n int) {
func (m *DeleteIndexMessage) Size() (n int) {
var l int
_ = l
l = len(m.DB)
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
return n
}
func (m *CreateDBMessage) Size() (n int) {
func (m *CreateIndexMessage) Size() (n int) {
var l int
_ = l
l = len(m.DB)
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
@ -1008,7 +1008,7 @@ func (m *CreateDBMessage) Size() (n int) {
func (m *CreateFrameMessage) Size() (n int) {
var l int
_ = l
l = len(m.DB)
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
@ -1026,7 +1026,7 @@ func (m *CreateFrameMessage) Size() (n int) {
func (m *DeleteFrameMessage) Size() (n int) {
var l int
_ = l
l = len(m.DB)
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
@ -1051,7 +1051,7 @@ func (m *Frame) Size() (n int) {
return n
}
func (m *DB) Size() (n int) {
func (m *Index) Size() (n int) {
var l int
_ = l
l = len(m.Name)
@ -1085,8 +1085,8 @@ func (m *NodeState) Size() (n int) {
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if len(m.DBs) > 0 {
for _, e := range m.DBs {
if len(m.Indexes) > 0 {
for _, e := range m.Indexes {
l = e.Size()
n += 1 + l + sovPrivate(uint64(l))
}
@ -1107,7 +1107,7 @@ func sovPrivate(x uint64) (n int) {
func sozPrivate(x uint64) (n int) {
return sovPrivate(uint64((x << 1) ^ uint64((int64(x) >> 63))))
}
func (m *DBMeta) Unmarshal(dAtA []byte) error {
func (m *IndexMeta) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -1130,10 +1130,10 @@ func (m *DBMeta) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DBMeta: wiretype end group for non-group")
return fmt.Errorf("proto: IndexMeta: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DBMeta: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: IndexMeta: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
@ -1501,7 +1501,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error {
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -1526,7 +1526,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DB = string(dAtA[iNdEx:postIndex])
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
@ -2118,7 +2118,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -2143,7 +2143,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DB = string(dAtA[iNdEx:postIndex])
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 0 {
@ -2185,7 +2185,7 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error {
func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -2208,15 +2208,15 @@ func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DeleteDBMessage: wiretype end group for non-group")
return fmt.Errorf("proto: DeleteIndexMessage: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DeleteDBMessage: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: DeleteIndexMessage: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -2241,7 +2241,7 @@ func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DB = string(dAtA[iNdEx:postIndex])
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
default:
iNdEx = preIndex
@ -2264,7 +2264,7 @@ func (m *DeleteDBMessage) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *CreateDBMessage) Unmarshal(dAtA []byte) error {
func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -2287,15 +2287,15 @@ func (m *CreateDBMessage) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: CreateDBMessage: wiretype end group for non-group")
return fmt.Errorf("proto: CreateIndexMessage: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: CreateDBMessage: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: CreateIndexMessage: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -2320,7 +2320,7 @@ func (m *CreateDBMessage) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DB = string(dAtA[iNdEx:postIndex])
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
@ -2349,7 +2349,7 @@ func (m *CreateDBMessage) Unmarshal(dAtA []byte) error {
return io.ErrUnexpectedEOF
}
if m.Meta == nil {
m.Meta = &DBMeta{}
m.Meta = &IndexMeta{}
}
if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
@ -2407,7 +2407,7 @@ func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error {
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -2432,7 +2432,7 @@ func (m *CreateFrameMessage) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DB = string(dAtA[iNdEx:postIndex])
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
@ -2548,7 +2548,7 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error {
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -2573,7 +2573,7 @@ func (m *DeleteFrameMessage) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DB = string(dAtA[iNdEx:postIndex])
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
@ -2737,7 +2737,7 @@ func (m *Frame) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *DB) Unmarshal(dAtA []byte) error {
func (m *Index) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -2760,10 +2760,10 @@ func (m *DB) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: DB: wiretype end group for non-group")
return fmt.Errorf("proto: Index: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: DB: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: Index: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
@ -2822,7 +2822,7 @@ func (m *DB) Unmarshal(dAtA []byte) error {
return io.ErrUnexpectedEOF
}
if m.Meta == nil {
m.Meta = &DBMeta{}
m.Meta = &IndexMeta{}
}
if err := m.Meta.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
@ -2988,7 +2988,7 @@ func (m *NodeState) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DBs", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Indexes", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
@ -3012,8 +3012,8 @@ func (m *NodeState) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DBs = append(m.DBs, &DB{})
if err := m.DBs[len(m.DBs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
m.Indexes = append(m.Indexes, &Index{})
if err := m.Indexes[len(m.Indexes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
@ -3146,43 +3146,43 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 596 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0x13, 0x41,
0x14, 0x76, 0x7f, 0x68, 0xe8, 0x41, 0x4a, 0x19, 0x8d, 0x59, 0x89, 0x69, 0xea, 0xc4, 0x08, 0xf1,
0x82, 0x0b, 0xbc, 0x31, 0xc4, 0xab, 0x65, 0x51, 0x9a, 0x00, 0x89, 0x03, 0x7a, 0x3f, 0x94, 0x13,
0xdd, 0xb0, 0xdd, 0xad, 0xbb, 0x53, 0xa0, 0xde, 0xfa, 0x12, 0x26, 0x3e, 0x83, 0xef, 0xe1, 0xa5,
0x8f, 0x60, 0xea, 0x8b, 0x98, 0x39, 0x33, 0xfb, 0x63, 0x29, 0x51, 0xef, 0xe6, 0x7c, 0xe7, 0xef,
0x9b, 0x6f, 0xbf, 0x59, 0x58, 0x1d, 0xe7, 0xf1, 0xa5, 0x54, 0xb8, 0x3d, 0xce, 0x33, 0x95, 0xb1,
0xe5, 0x38, 0x55, 0x98, 0xa7, 0x32, 0xe1, 0x87, 0xd0, 0x8a, 0xc2, 0x23, 0x54, 0x92, 0xf5, 0x61,
0x65, 0x2f, 0x4b, 0x26, 0xa3, 0xf4, 0x50, 0x9e, 0x61, 0x12, 0x38, 0x7d, 0x67, 0xab, 0x2d, 0x9a,
0x90, 0xae, 0x38, 0x8d, 0x47, 0xf8, 0x66, 0x22, 0x53, 0x35, 0x19, 0x05, 0xae, 0xa9, 0x68, 0x40,
0xfc, 0x9b, 0x03, 0xed, 0x57, 0xb9, 0x1c, 0x21, 0x4d, 0xdc, 0x80, 0x65, 0x91, 0x5d, 0x35, 0xc7,
0x55, 0x31, 0x7b, 0x0a, 0x9d, 0x41, 0x7a, 0x89, 0x79, 0x81, 0xfb, 0xa9, 0x3c, 0x4b, 0xf0, 0x9c,
0xc6, 0x2d, 0x8b, 0x39, 0x94, 0x3d, 0x82, 0xf6, 0x9e, 0x1c, 0x7e, 0xc0, 0xd3, 0xe9, 0x18, 0x03,
0x8f, 0x86, 0xd4, 0x40, 0x95, 0x3d, 0x89, 0x3f, 0x61, 0xe0, 0xf7, 0x9d, 0xad, 0x55, 0x51, 0x03,
0xf3, 0x7c, 0x97, 0x6e, 0xf2, 0xe5, 0xd0, 0x19, 0x8c, 0xc6, 0x59, 0xae, 0x04, 0x16, 0xe3, 0x2c,
0x2d, 0x90, 0x75, 0xc1, 0xdb, 0xcf, 0x73, 0x4b, 0x57, 0x1f, 0xf9, 0x35, 0x74, 0xc3, 0x24, 0x1b,
0x5e, 0x44, 0x52, 0x49, 0x81, 0x1f, 0x27, 0x58, 0x28, 0xd6, 0x01, 0x37, 0x0a, 0x6d, 0x91, 0x1b,
0x85, 0xec, 0x3e, 0x2c, 0xd1, 0xb5, 0xad, 0x26, 0x26, 0xd0, 0x28, 0x75, 0x12, 0x6f, 0x5f, 0x98,
0x40, 0xa3, 0x27, 0x49, 0x3c, 0x34, 0x7c, 0x7d, 0x61, 0x02, 0xc6, 0xc0, 0x7f, 0x17, 0xe3, 0x95,
0x25, 0x49, 0x67, 0x3e, 0x80, 0xf5, 0xc6, 0x66, 0x4b, 0xf0, 0x01, 0xb4, 0x44, 0x76, 0x35, 0x88,
0x8a, 0xc0, 0xe9, 0x7b, 0x5b, 0xbe, 0xb0, 0x11, 0x49, 0x41, 0xdf, 0x4a, 0xa7, 0x5c, 0x4a, 0xd5,
0x00, 0x7f, 0x08, 0x4b, 0xa4, 0x8b, 0xbe, 0x5f, 0xdd, 0xab, 0x8f, 0xfc, 0xab, 0x03, 0xeb, 0x47,
0xf2, 0x9a, 0x68, 0x14, 0xd5, 0x9a, 0x03, 0x68, 0x57, 0x20, 0x55, 0xaf, 0xec, 0x3c, 0xdb, 0x2e,
0x5d, 0xb3, 0x7d, 0xa3, 0xbe, 0x46, 0xf6, 0x53, 0x95, 0x4f, 0x45, 0xdd, 0xbc, 0xf1, 0x12, 0x3a,
0x7f, 0x26, 0x35, 0x87, 0x0b, 0x9c, 0x96, 0x1a, 0x5f, 0xe0, 0x54, 0x6b, 0x72, 0x29, 0x93, 0x89,
0xd1, 0xcf, 0x17, 0x26, 0xd8, 0x75, 0x5f, 0x38, 0x7c, 0x17, 0xd8, 0x5e, 0x8e, 0x52, 0x21, 0x0d,
0x38, 0xc2, 0xa2, 0x90, 0xef, 0x71, 0x91, 0xfe, 0x46, 0x53, 0xb7, 0xa1, 0x29, 0x7f, 0x0c, 0x6b,
0x11, 0x26, 0xa8, 0x50, 0x3b, 0x7c, 0x61, 0x23, 0x7f, 0x0d, 0x6b, 0x66, 0xfc, 0xad, 0x25, 0xec,
0x09, 0xf8, 0xda, 0xcd, 0x34, 0x7a, 0x65, 0xa7, 0x5b, 0x8b, 0x60, 0xde, 0x8d, 0xa0, 0x2c, 0x1f,
0x96, 0x3c, 0xad, 0xfd, 0x6f, 0xe5, 0xb9, 0xc0, 0x27, 0x9b, 0x76, 0x83, 0x47, 0x1b, 0xee, 0xd5,
0x1b, 0xaa, 0xa7, 0x64, 0x97, 0xec, 0x02, 0x33, 0x17, 0xfa, 0xff, 0x25, 0x3c, 0xb2, 0xa8, 0x76,
0xda, 0xb1, 0xce, 0x9a, 0x06, 0x3a, 0x57, 0x0c, 0xdc, 0xbf, 0x31, 0xf8, 0xec, 0xe8, 0x65, 0x0b,
0x67, 0xfc, 0x93, 0x4e, 0xfa, 0x9f, 0x50, 0xba, 0xc1, 0x3e, 0x8b, 0x2a, 0x66, 0x9b, 0xd0, 0xa2,
0x7d, 0x45, 0xe0, 0x93, 0xe1, 0xd6, 0xe6, 0x78, 0x08, 0x9b, 0xe6, 0x6f, 0xa1, 0x7d, 0x9c, 0x9d,
0xe3, 0x89, 0x92, 0x8a, 0xee, 0x73, 0x90, 0x15, 0xaa, 0xe4, 0xa2, 0xcf, 0xe4, 0x07, 0x9d, 0x2c,
0x25, 0x30, 0x95, 0x3d, 0xf0, 0xa2, 0xb0, 0x08, 0x3c, 0x1a, 0x7e, 0xb7, 0x49, 0x50, 0xe8, 0x44,
0xd8, 0xfd, 0x3e, 0xeb, 0x39, 0x3f, 0x66, 0x3d, 0xe7, 0xe7, 0xac, 0xe7, 0x7c, 0xf9, 0xd5, 0xbb,
0x73, 0xd6, 0xa2, 0xdf, 0xe5, 0xf3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe6, 0xcf, 0x20, 0xf1,
0x3f, 0x05, 0x00, 0x00,
// 594 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x9c, 0x54, 0xc1, 0x6e, 0xd3, 0x4c,
0x10, 0xfe, 0x9d, 0xb8, 0xfd, 0xe3, 0x89, 0x1a, 0xd2, 0x05, 0x21, 0x53, 0xa1, 0x28, 0xda, 0x03,
0x0d, 0x3d, 0xe4, 0x50, 0x2e, 0x08, 0x71, 0xa8, 0x9a, 0x04, 0x35, 0x12, 0x29, 0x62, 0x53, 0x71,
0x66, 0x93, 0x8c, 0xc0, 0x8a, 0x63, 0x07, 0x7b, 0x93, 0x34, 0x1c, 0xb8, 0xf3, 0x06, 0x48, 0x3c,
0x03, 0xef, 0xc1, 0x91, 0x47, 0x40, 0xe1, 0x45, 0xd0, 0x8e, 0xd7, 0x76, 0x70, 0x29, 0x15, 0xdc,
0x76, 0xbe, 0x99, 0x9d, 0xef, 0x9b, 0xcf, 0xb3, 0x86, 0xbd, 0x79, 0xe4, 0x2d, 0xa5, 0xc2, 0xf6,
0x3c, 0x0a, 0x55, 0xc8, 0x2a, 0x5e, 0xa0, 0x30, 0x0a, 0xa4, 0xcf, 0x5f, 0x80, 0xd3, 0x0f, 0x26,
0x78, 0x39, 0x40, 0x25, 0x59, 0x13, 0xaa, 0x9d, 0xd0, 0x5f, 0xcc, 0x82, 0xe7, 0x72, 0x84, 0xbe,
0x6b, 0x35, 0xad, 0x96, 0x23, 0xb6, 0x21, 0x5d, 0x71, 0xe1, 0xcd, 0xf0, 0xe5, 0x42, 0x06, 0x6a,
0x31, 0x73, 0x4b, 0x49, 0xc5, 0x16, 0xc4, 0xbf, 0x58, 0xe0, 0x3c, 0x8b, 0xe4, 0x0c, 0xa9, 0xe3,
0x01, 0x54, 0x44, 0xb8, 0xda, 0x6e, 0x97, 0xc5, 0xec, 0x01, 0xd4, 0xfa, 0xc1, 0x12, 0xa3, 0x18,
0x7b, 0x81, 0x1c, 0xf9, 0x38, 0xa1, 0x76, 0x15, 0x51, 0x40, 0xd9, 0x7d, 0x70, 0x3a, 0x72, 0xfc,
0x16, 0x2f, 0xd6, 0x73, 0x74, 0xcb, 0xd4, 0x24, 0x07, 0xb2, 0xec, 0xd0, 0x7b, 0x8f, 0xae, 0xdd,
0xb4, 0x5a, 0x7b, 0x22, 0x07, 0x8a, 0x7a, 0x77, 0xae, 0xea, 0xe5, 0x50, 0xeb, 0xcf, 0xe6, 0x61,
0xa4, 0x04, 0xc6, 0xf3, 0x30, 0x88, 0x91, 0xd5, 0xa1, 0xdc, 0x8b, 0x22, 0x23, 0x57, 0x1f, 0xf9,
0x07, 0xa8, 0x9f, 0xfa, 0xe1, 0x78, 0xda, 0x95, 0x4a, 0x0a, 0x7c, 0xb7, 0xc0, 0x58, 0xb1, 0x3b,
0xb0, 0x43, 0xc6, 0x99, 0xba, 0x24, 0xd0, 0x28, 0x0d, 0x6f, 0x9c, 0x49, 0x02, 0x8d, 0xd2, 0x7d,
0x52, 0x6f, 0x8b, 0x24, 0xd0, 0xe8, 0xd0, 0xf7, 0xc6, 0x89, 0x6a, 0x5b, 0x24, 0x01, 0x63, 0x60,
0xbf, 0xf2, 0x70, 0x65, 0xa4, 0xd2, 0x99, 0xf7, 0x61, 0x7f, 0x8b, 0xdf, 0xc8, 0xbc, 0x0b, 0xbb,
0x22, 0x5c, 0xf5, 0xbb, 0xb1, 0x6b, 0x35, 0xcb, 0x2d, 0x5b, 0x98, 0x88, 0x0c, 0xa1, 0x2f, 0xa6,
0x53, 0x25, 0x4a, 0xe5, 0x00, 0xbf, 0x07, 0x3b, 0xe4, 0x8e, 0x9e, 0x32, 0xbf, 0xab, 0x8f, 0xfc,
0xb3, 0x05, 0xfb, 0x03, 0x79, 0x49, 0x32, 0xe2, 0x8c, 0xe6, 0x0c, 0x9c, 0x0c, 0xa4, 0xea, 0xea,
0xf1, 0x51, 0x3b, 0x5d, 0x9f, 0xf6, 0x95, 0xfa, 0x1c, 0xe9, 0x05, 0x2a, 0x5a, 0x8b, 0xfc, 0xf2,
0xc1, 0x53, 0xa8, 0xfd, 0x9a, 0xd4, 0x1a, 0xa6, 0xb8, 0x4e, 0x9d, 0x9e, 0xe2, 0x5a, 0x7b, 0xb2,
0x94, 0xfe, 0x22, 0xf1, 0xcf, 0x16, 0x49, 0xf0, 0xa4, 0xf4, 0xd8, 0xe2, 0x27, 0xc0, 0x3a, 0x11,
0x4a, 0x85, 0xd4, 0x60, 0x80, 0x71, 0x2c, 0xdf, 0xe0, 0xf5, 0x5f, 0x21, 0x71, 0xb6, 0xb4, 0xe5,
0x2c, 0x3f, 0x02, 0xd6, 0x45, 0x1f, 0x15, 0x9a, 0x85, 0xff, 0x43, 0x07, 0x3e, 0x4c, 0xd9, 0x6e,
0xae, 0x65, 0x87, 0x60, 0xeb, 0x5d, 0x27, 0xb2, 0xea, 0xf1, 0xed, 0xdc, 0x9c, 0xec, 0x61, 0x09,
0x2a, 0xe0, 0x5e, 0xda, 0xd4, 0xbc, 0x8f, 0x1b, 0x46, 0xf8, 0xcd, 0x22, 0xa5, 0x54, 0xe5, 0x22,
0x55, 0xf6, 0xe2, 0x0c, 0xd5, 0x49, 0x3a, 0xeb, 0xbf, 0x52, 0xf1, 0xae, 0x41, 0xf5, 0x42, 0x9e,
0xeb, 0x6c, 0x72, 0x87, 0xce, 0xd7, 0x8f, 0x5c, 0xd4, 0xf1, 0xd1, 0x32, 0x94, 0x7f, 0xd7, 0xa6,
0xe0, 0x9c, 0xfe, 0x8d, 0xa4, 0xab, 0x63, 0xde, 0x50, 0x16, 0xb3, 0x43, 0xd8, 0x25, 0xd6, 0xd8,
0xb5, 0x69, 0x3b, 0x6f, 0x15, 0xd4, 0x08, 0x93, 0xe6, 0xaf, 0xc1, 0x39, 0x0f, 0x27, 0x38, 0x54,
0x52, 0xd1, 0x54, 0x67, 0x61, 0xac, 0x52, 0x39, 0xfa, 0x4c, 0x6b, 0xa3, 0x93, 0xa9, 0x11, 0x49,
0xe5, 0x43, 0xf8, 0x9f, 0xe4, 0x60, 0xec, 0x96, 0x8b, 0x04, 0x94, 0x10, 0x69, 0xfe, 0xb4, 0xfe,
0x75, 0xd3, 0xb0, 0xbe, 0x6d, 0x1a, 0xd6, 0xf7, 0x4d, 0xc3, 0xfa, 0xf4, 0xa3, 0xf1, 0xdf, 0x68,
0x97, 0xfe, 0xb7, 0x8f, 0x7e, 0x06, 0x00, 0x00, 0xff, 0xff, 0xc6, 0x68, 0xdc, 0x63, 0x80, 0x05,
0x00, 0x00,
}

View file

@ -2,7 +2,7 @@ syntax = "proto3";
package internal;
message DBMeta {
message IndexMeta {
string ColumnLabel = 1;
string TimeQuantum = 2;
}
@ -20,7 +20,7 @@ message ImportResponse {
}
message BlockDataRequest {
string DB = 1;
string Index = 1;
string Frame = 2;
string View = 5;
uint64 Slice = 4;
@ -41,27 +41,27 @@ message MaxSlicesResponse {
}
message CreateSliceMessage {
string DB = 1;
string Index = 1;
uint64 Slice = 2;
}
message DeleteDBMessage {
string DB = 1;
message DeleteIndexMessage {
string Index = 1;
}
message CreateDBMessage {
string DB = 1;
DBMeta Meta = 2;
message CreateIndexMessage {
string Index = 1;
IndexMeta Meta = 2;
}
message CreateFrameMessage {
string DB = 1;
string Index = 1;
string Frame = 2;
FrameMeta Meta = 3;
}
message DeleteFrameMessage {
string DB = 1;
string Index = 1;
string Frame = 2;
}
@ -70,9 +70,9 @@ message Frame {
FrameMeta Meta = 2;
}
message DB {
message Index {
string Name = 1;
DBMeta Meta = 2;
IndexMeta Meta = 2;
uint64 MaxSlice = 3;
repeated Frame Frames = 4;
}
@ -80,5 +80,5 @@ message DB {
message NodeState {
string Host = 1;
string State = 2;
repeated DB DBs = 3;
repeated Index Indexes = 3;
}

View file

@ -189,7 +189,7 @@ func (m *QueryResult) GetPairs() []*Pair {
}
type ImportRequest struct {
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"`
@ -627,11 +627,11 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.DB) > 0 {
if len(m.Index) > 0 {
dAtA[i] = 0xa
i++
i = encodeVarintPublic(dAtA, i, uint64(len(m.DB)))
i += copy(dAtA[i:], m.DB)
i = encodeVarintPublic(dAtA, i, uint64(len(m.Index)))
i += copy(dAtA[i:], m.Index)
}
if len(m.Frame) > 0 {
dAtA[i] = 0x12
@ -899,7 +899,7 @@ func (m *QueryResult) Size() (n int) {
func (m *ImportRequest) Size() (n int) {
var l int
_ = l
l = len(m.DB)
l = len(m.Index)
if l > 0 {
n += 1 + l + sovPublic(uint64(l))
}
@ -2185,7 +2185,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
switch fieldNum {
case 1:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
}
var stringLen uint64
for shift := uint(0); ; shift += 7 {
@ -2210,7 +2210,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.DB = string(dAtA[iNdEx:postIndex])
m.Index = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 2:
if wireType != 2 {
@ -2575,42 +2575,41 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 579 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x40,
0x10, 0x66, 0x6d, 0x27, 0x4d, 0x26, 0x6d, 0x14, 0xad, 0xf8, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x83,
0x4f, 0xa9, 0x54, 0x1e, 0x00, 0xe1, 0x24, 0x95, 0x22, 0x44, 0x45, 0x27, 0x85, 0xbb, 0x5b, 0x56,
0xc5, 0x92, 0xff, 0x58, 0xaf, 0x85, 0xf2, 0x00, 0xdc, 0x91, 0xb8, 0x70, 0xe5, 0xc6, 0xa3, 0x70,
0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0xbb, 0xde, 0xd8, 0xe5, 0x80, 0xb8, 0xed, 0xf7, 0xcd, 0xce,
0x7a, 0xbe, 0xf9, 0x66, 0x0c, 0xc7, 0x55, 0x73, 0x9d, 0xa5, 0x37, 0x8b, 0x4a, 0x96, 0xaa, 0xe4,
0xa3, 0xb4, 0x50, 0x42, 0x16, 0x49, 0x16, 0xc6, 0x30, 0x8c, 0x53, 0x95, 0x27, 0x15, 0xe7, 0xe0,
0xc5, 0xa9, 0xaa, 0x7d, 0x16, 0xb8, 0x91, 0x87, 0xfa, 0xcc, 0x9f, 0xc2, 0xe0, 0x85, 0x52, 0xb2,
0xf6, 0x9d, 0xc0, 0x8d, 0x26, 0x67, 0xd3, 0x85, 0xcd, 0x5b, 0x10, 0x8d, 0x26, 0x18, 0x2e, 0xc0,
0x7b, 0x9d, 0xa4, 0x92, 0xcf, 0xc0, 0x7d, 0x29, 0x76, 0x3e, 0x0b, 0x58, 0xe4, 0x21, 0x1d, 0xf9,
0x7d, 0x18, 0x2c, 0xcb, 0xa6, 0x50, 0xbe, 0xa3, 0x39, 0x03, 0xc2, 0x37, 0xe0, 0xc6, 0xa9, 0xa2,
0x20, 0x96, 0x1f, 0x37, 0xab, 0x36, 0xc1, 0x00, 0xfe, 0x18, 0x46, 0xcb, 0x32, 0x6b, 0xf2, 0x62,
0xb3, 0x6a, 0xb3, 0x0e, 0x98, 0x3f, 0x81, 0xf1, 0x55, 0x9a, 0x8b, 0x5a, 0x25, 0x79, 0xe5, 0xbb,
0x01, 0x8b, 0x5c, 0xec, 0x88, 0x70, 0x0d, 0x27, 0xe6, 0x26, 0x55, 0xb5, 0x15, 0x8a, 0x4f, 0xc1,
0x39, 0xbc, 0xee, 0x6c, 0x56, 0xff, 0xa9, 0xe6, 0x3b, 0x03, 0x8f, 0x4e, 0x7d, 0x39, 0x63, 0x23,
0x87, 0x83, 0x77, 0xb5, 0xab, 0x44, 0x5b, 0x97, 0x3e, 0xf3, 0x00, 0x26, 0x5b, 0x25, 0xd3, 0xe2,
0xf6, 0x6d, 0x92, 0x35, 0x42, 0x57, 0x35, 0xc6, 0x3e, 0x45, 0x8a, 0x36, 0x85, 0x32, 0x61, 0x4f,
0x17, 0x7d, 0xc0, 0xa4, 0x28, 0x2e, 0xcb, 0xcc, 0x04, 0x07, 0x01, 0x8b, 0x46, 0xd8, 0x11, 0x7c,
0x0e, 0x70, 0x9e, 0x95, 0x49, 0x9b, 0x3b, 0x0c, 0x58, 0xc4, 0xb0, 0xc7, 0x84, 0xa7, 0x70, 0x44,
0x95, 0xbe, 0x4a, 0xaa, 0x4e, 0x1b, 0xfb, 0x97, 0xb6, 0xcf, 0x0c, 0x8e, 0x2f, 0x1b, 0x21, 0x77,
0x28, 0x3e, 0x34, 0xa2, 0xd6, 0x1e, 0x68, 0xdc, 0xaa, 0x34, 0x80, 0x3f, 0x84, 0xe1, 0x36, 0x4b,
0x6f, 0x84, 0xe9, 0x94, 0x87, 0x2d, 0x22, 0xad, 0x5d, 0x87, 0x6b, 0xad, 0x75, 0x84, 0x7d, 0x8a,
0xfb, 0x70, 0x74, 0xd9, 0x24, 0x85, 0x6a, 0x72, 0x2d, 0x75, 0x8c, 0x16, 0xd2, 0x9b, 0x28, 0xf2,
0x52, 0x59, 0x99, 0x2d, 0x0a, 0xbf, 0x30, 0x38, 0x69, 0x4b, 0xaa, 0xab, 0xb2, 0xa8, 0x05, 0xf5,
0x7d, 0x2d, 0xa5, 0xed, 0xfb, 0x5a, 0x4a, 0x7e, 0x0a, 0x47, 0x28, 0xea, 0x26, 0x53, 0xd6, 0xba,
0x07, 0x9d, 0x3c, 0x9b, 0xdb, 0x64, 0x0a, 0xed, 0x2d, 0xfe, 0x1c, 0xa6, 0x77, 0x46, 0x81, 0x6a,
0xa5, 0xbc, 0x47, 0x5d, 0xde, 0x9d, 0x38, 0xfe, 0x75, 0x3d, 0xfc, 0xc4, 0x60, 0xd2, 0x7b, 0x99,
0x47, 0x76, 0x4d, 0x74, 0x59, 0x93, 0xb3, 0x59, 0xf7, 0x90, 0xe1, 0xd1, 0xae, 0xd1, 0x31, 0xb0,
0x8b, 0x76, 0x40, 0xd8, 0x05, 0xd9, 0x42, 0xab, 0x61, 0xbf, 0xdf, 0xb3, 0x85, 0x68, 0x34, 0x41,
0xea, 0xda, 0xf2, 0x7d, 0x52, 0xdc, 0x8a, 0x77, 0xba, 0x6b, 0x23, 0xb4, 0x30, 0xfc, 0xc6, 0xe0,
0x64, 0x93, 0x57, 0xa5, 0x54, 0xd6, 0xb1, 0x29, 0x38, 0xab, 0xb8, 0x6d, 0x8e, 0xb3, 0x8a, 0xc9,
0xc1, 0x73, 0x99, 0xe4, 0x66, 0x28, 0xc7, 0x68, 0x00, 0xb1, 0xda, 0x33, 0xed, 0x91, 0x87, 0x06,
0x68, 0x0f, 0x68, 0xc9, 0x6a, 0xdf, 0x33, 0xbe, 0x1a, 0x44, 0x53, 0x68, 0x77, 0xac, 0xf6, 0x07,
0x3a, 0xd4, 0x11, 0x34, 0x85, 0x87, 0x25, 0xab, 0xfd, 0x61, 0xe0, 0x46, 0x2e, 0xf6, 0x98, 0x78,
0xf6, 0x63, 0x3f, 0x67, 0x3f, 0xf7, 0x73, 0xf6, 0x6b, 0x3f, 0x67, 0x5f, 0x7f, 0xcf, 0xef, 0x5d,
0x0f, 0xf5, 0x5f, 0xe6, 0xd9, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xaf, 0x79, 0x70, 0xf4, 0x75,
0x04, 0x00, 0x00,
// 576 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40,
0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57,
0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xa6, 0x33, 0xb0, 0xf7, 0xcc, 0xb4,
0x06, 0x4b, 0xfe, 0xd1, 0xdd, 0x16, 0xe4, 0x00, 0xec, 0x91, 0xd8, 0x70, 0x03, 0x38, 0x0a, 0x4b,
0x8e, 0x80, 0xc2, 0x45, 0x50, 0x75, 0xbb, 0x63, 0x0f, 0x0b, 0xc4, 0xae, 0xdf, 0xab, 0xae, 0x76,
0xbd, 0x7a, 0x55, 0x86, 0x69, 0x55, 0x5f, 0x65, 0xe9, 0xf5, 0xaa, 0x12, 0xa5, 0x2a, 0xe9, 0x28,
0x2d, 0x14, 0x17, 0x45, 0x92, 0x05, 0x11, 0x0c, 0xa2, 0x54, 0xe5, 0x49, 0x45, 0x29, 0xb8, 0x51,
0xaa, 0xa4, 0x47, 0x7c, 0x27, 0x74, 0x99, 0x3e, 0xd3, 0xa7, 0xd0, 0x7f, 0xa1, 0x94, 0x90, 0x5e,
0xcf, 0x77, 0xc2, 0xc9, 0xe9, 0x7c, 0x65, 0xf3, 0x56, 0x48, 0x33, 0x13, 0x0c, 0x56, 0xe0, 0xbe,
0x4e, 0x52, 0x41, 0x17, 0xe0, 0xbc, 0xe4, 0x7b, 0x8f, 0xf8, 0x24, 0x74, 0x19, 0x1e, 0xe9, 0x7d,
0xe8, 0xaf, 0xcb, 0xba, 0x50, 0x5e, 0x4f, 0x73, 0x06, 0x04, 0x6f, 0xc0, 0x89, 0x52, 0x85, 0x41,
0x56, 0x7e, 0x88, 0x37, 0x4d, 0x82, 0x01, 0xf4, 0x31, 0x8c, 0xd6, 0x65, 0x56, 0xe7, 0x45, 0xbc,
0x69, 0xb2, 0x8e, 0x98, 0x3e, 0x81, 0xf1, 0x65, 0x9a, 0x73, 0xa9, 0x92, 0xbc, 0xf2, 0x1c, 0x9f,
0x84, 0x0e, 0x6b, 0x89, 0x60, 0x0b, 0x33, 0x73, 0x13, 0xab, 0xda, 0x71, 0x45, 0xe7, 0xd0, 0x3b,
0xbe, 0xde, 0x8b, 0x37, 0xff, 0xa9, 0xe6, 0x3b, 0x01, 0x17, 0x4f, 0x5d, 0x39, 0x63, 0x23, 0x87,
0x82, 0x7b, 0xb9, 0xaf, 0x78, 0x53, 0x97, 0x3e, 0x53, 0x1f, 0x26, 0x3b, 0x25, 0xd2, 0xe2, 0xf6,
0x6d, 0x92, 0xd5, 0x5c, 0x57, 0x35, 0x66, 0x5d, 0x0a, 0x15, 0xc5, 0x85, 0x32, 0x61, 0x57, 0x17,
0x7d, 0xc4, 0xa8, 0x28, 0x2a, 0xcb, 0xcc, 0x04, 0xfb, 0x3e, 0x09, 0x47, 0xac, 0x25, 0xe8, 0x12,
0xe0, 0x2c, 0x2b, 0x93, 0x26, 0x77, 0xe0, 0x93, 0x90, 0xb0, 0x0e, 0x13, 0x9c, 0xc0, 0x10, 0x2b,
0x7d, 0x95, 0x54, 0xad, 0x36, 0xf2, 0x2f, 0x6d, 0x9f, 0x09, 0x4c, 0x2f, 0x6a, 0x2e, 0xf6, 0x8c,
0xbf, 0xaf, 0xb9, 0xd4, 0x1e, 0x68, 0xdc, 0xa8, 0x34, 0x80, 0x3e, 0x84, 0xc1, 0x2e, 0x4b, 0xaf,
0xb9, 0xe9, 0x94, 0xcb, 0x1a, 0x84, 0x5a, 0xdb, 0x0e, 0x4b, 0xad, 0x75, 0xc4, 0xba, 0x14, 0xf5,
0x60, 0x78, 0x51, 0x27, 0x85, 0xaa, 0x73, 0x2d, 0x75, 0xcc, 0x2c, 0xc4, 0x37, 0x19, 0xcf, 0x4b,
0x65, 0x65, 0x36, 0x28, 0xf8, 0x42, 0x60, 0xd6, 0x94, 0x24, 0xab, 0xb2, 0x90, 0x1c, 0xfb, 0xbe,
0x15, 0xc2, 0xf6, 0x7d, 0x2b, 0x04, 0x3d, 0x81, 0x21, 0xe3, 0xb2, 0xce, 0x94, 0xb5, 0xee, 0x41,
0x2b, 0xcf, 0xe6, 0xd6, 0x99, 0x62, 0xf6, 0x16, 0x7d, 0x0e, 0xf3, 0x3b, 0xa3, 0x80, 0xb5, 0x62,
0xde, 0xa3, 0x36, 0xef, 0x4e, 0x9c, 0xfd, 0x75, 0x3d, 0xf8, 0x44, 0x60, 0xd2, 0x79, 0x99, 0x86,
0x76, 0x4d, 0x74, 0x59, 0x93, 0xd3, 0x45, 0xfb, 0x90, 0xe1, 0x99, 0x5d, 0xa3, 0x29, 0x90, 0xf3,
0x66, 0x40, 0xc8, 0x39, 0xda, 0x82, 0xab, 0x61, 0xbf, 0xdf, 0xb1, 0x05, 0x69, 0x66, 0x82, 0xd8,
0xb5, 0xf5, 0xbb, 0xa4, 0xb8, 0xe5, 0x37, 0xba, 0x6b, 0x23, 0x66, 0x61, 0xf0, 0x8d, 0xc0, 0x2c,
0xce, 0xab, 0x52, 0xa8, 0x8e, 0x63, 0x71, 0x71, 0xc3, 0x3f, 0x5a, 0xc7, 0x34, 0x40, 0xf6, 0x4c,
0x24, 0xb9, 0x19, 0xcd, 0x31, 0x33, 0x00, 0x59, 0xed, 0x9c, 0x76, 0xca, 0x65, 0x06, 0x68, 0x27,
0x70, 0xd5, 0xa4, 0xe7, 0x1a, 0x77, 0x0d, 0xc2, 0x59, 0xb4, 0x9b, 0x26, 0xbd, 0xbe, 0x0e, 0xb5,
0x04, 0xce, 0xe2, 0x71, 0xd5, 0xa4, 0x37, 0xf0, 0x9d, 0xd0, 0x61, 0x1d, 0x26, 0x5a, 0xfc, 0x38,
0x2c, 0xc9, 0xcf, 0xc3, 0x92, 0xfc, 0x3a, 0x2c, 0xc9, 0xd7, 0xdf, 0xcb, 0x7b, 0x57, 0x03, 0xfd,
0xaf, 0x79, 0xf6, 0x27, 0x00, 0x00, 0xff, 0xff, 0x48, 0x20, 0x4b, 0xe7, 0x7b, 0x04, 0x00, 0x00,
}

View file

@ -58,7 +58,7 @@ message QueryResult {
}
message ImportRequest {
string DB = 1;
string Index = 1;
string Frame = 2;
uint64 Slice = 3;
repeated uint64 RowIDs = 4;

View file

@ -11,9 +11,9 @@ import (
var (
ErrHostRequired = errors.New("host required")
ErrDatabaseRequired = errors.New("database required")
ErrDatabaseExists = errors.New("database already exists")
ErrDatabaseNotFound = errors.New("database not found")
ErrIndexRequired = errors.New("index required")
ErrIndexExists = errors.New("index already exists")
ErrIndexNotFound = errors.New("index not found")
// ErrFrameRequired is returned when no frame is specified.
ErrFrameRequired = errors.New("frame required")
@ -24,18 +24,18 @@ var (
ErrInvalidView = errors.New("invalid view")
ErrInvalidCacheType = errors.New("invalid cache type")
ErrName = errors.New("invalid database or frame's name, must match [a-z0-9_-]")
ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]")
// ErrFragmentNotFound is returned when a fragment does not exist.
ErrFragmentNotFound = errors.New("fragment not found")
ErrQueryRequired = errors.New("query required")
)
// Regular expression to valuate db and frame's name
// Regular expression to valuate index and frame's name
// Todo: remove . when frame doesn't require . for topN
var nameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,64}$`)
// ColumnAttrSet represents a set of attributes for a vertical column in a database.
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
// Can have a set of attributes attached to it.
type ColumnAttrSet struct {
ID uint64 `json:"id"`

View file

@ -219,16 +219,16 @@ func (s *Server) monitorMaxSlices() {
for _, node := range s.Cluster.Nodes {
if s.Host != node.Host {
maxSlices, _ := checkMaxSlices(node.Host)
for db, newmax := range maxSlices {
// if we don't know about a db locally, log an error because
// db's should be created and synced prior to slice creation
if localdb := s.Holder.DB(db); localdb != nil {
if newmax > oldmaxslices[db] {
oldmaxslices[db] = newmax
localdb.SetRemoteMaxSlice(newmax)
for index, newmax := range maxSlices {
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex := s.Holder.Index(index); localIndex != nil {
if newmax > oldmaxslices[index] {
oldmaxslices[index] = newmax
localIndex.SetRemoteMaxSlice(newmax)
}
} else {
s.logger().Printf("Local DB not found: %s", db)
s.logger().Printf("Local Index not found: %s", index)
}
}
}
@ -240,31 +240,31 @@ func (s *Server) monitorMaxSlices() {
func (s *Server) ReceiveMessage(pb proto.Message) error {
switch obj := pb.(type) {
case *internal.CreateSliceMessage:
d := s.Holder.DB(obj.DB)
d := s.Holder.Index(obj.Index)
if d == nil {
return fmt.Errorf("Local DB not found: %s", obj.DB)
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
d.SetRemoteMaxSlice(obj.Slice)
case *internal.CreateDBMessage:
opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel}
_, err := s.Holder.CreateDB(obj.DB, opt)
case *internal.CreateIndexMessage:
opt := IndexOptions{ColumnLabel: obj.Meta.ColumnLabel}
_, err := s.Holder.CreateIndex(obj.Index, opt)
if err != nil {
return err
}
case *internal.DeleteDBMessage:
if err := s.Holder.DeleteDB(obj.DB); err != nil {
case *internal.DeleteIndexMessage:
if err := s.Holder.DeleteIndex(obj.Index); err != nil {
return err
}
case *internal.CreateFrameMessage:
db := s.Holder.DB(obj.DB)
index := s.Holder.Index(obj.Index)
opt := FrameOptions{RowLabel: obj.Meta.RowLabel}
_, err := db.CreateFrame(obj.Frame, opt)
_, err := index.CreateFrame(obj.Frame, opt)
if err != nil {
return err
}
case *internal.DeleteFrameMessage:
db := s.Holder.DB(obj.DB)
if err := db.DeleteFrame(obj.Frame); err != nil {
index := s.Holder.Index(obj.Index)
if err := index.DeleteFrame(obj.Frame); err != nil {
return err
}
}
@ -273,16 +273,16 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
// Server implements gossip.StateHandler.
// LocalState returns the state of the local node as well as the
// holder (dbs/frames) according to the local node.
// holder (indexes/frames) according to the local node.
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalState() (proto.Message, error) {
if s.Holder == nil {
return nil, errors.New("Server.Holder is nil.")
}
return &internal.NodeState{
Host: s.Host,
State: "OK", // TODO: make this work, pull from s.Cluster.Node
DBs: encodeDBs(s.Holder.DBs()),
Host: s.Host,
State: "OK", // TODO: make this work, pull from s.Cluster.Node
Indexes: encodeIndexes(s.Holder.Indexes()),
}, nil
}
@ -294,18 +294,18 @@ func (s *Server) HandleRemoteState(pb proto.Message) error {
func (s *Server) mergeRemoteState(ns *internal.NodeState) error {
// TODO: update some node state value in the cluster (it should be in cluster.node i guess)
// Create databases that don't exist.
for _, db := range ns.DBs {
opt := DBOptions{
ColumnLabel: db.Meta.ColumnLabel,
TimeQuantum: TimeQuantum(db.Meta.TimeQuantum),
// Create indexes that don't exist.
for _, index := range ns.Indexes {
opt := IndexOptions{
ColumnLabel: index.Meta.ColumnLabel,
TimeQuantum: TimeQuantum(index.Meta.TimeQuantum),
}
d, err := s.Holder.CreateDBIfNotExists(db.Name, opt)
d, err := s.Holder.CreateIndexIfNotExists(index.Name, opt)
if err != nil {
return err
}
// Create frames that don't exist.
for _, f := range db.Frames {
for _, f := range index.Frames {
opt := FrameOptions{
RowLabel: f.Meta.RowLabel,
TimeQuantum: TimeQuantum(f.Meta.TimeQuantum),

View file

@ -39,13 +39,13 @@ func TestMain_Set_Quick(t *testing.T) {
// Execute SetBit() commands.
for _, cmd := range cmds {
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
}
if err := client.CreateFrame(context.Background(), "d", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists {
if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists {
t.Fatal(err)
}
if _, err := m.Query("d", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil {
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil {
t.Fatal(err)
}
}
@ -61,7 +61,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("d", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -84,7 +84,7 @@ func TestMain_Set_Quick(t *testing.T) {
},
},
}) + "\n"
if res, err := m.Query("d", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
t.Fatal(err)
} else if res != exp {
t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp)
@ -109,47 +109,47 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Create frames.
client := m.Client()
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "z", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "neg", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Set bits on different rows in different frames.
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil {
if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=2, frame="x.n", columnID=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(id=2, frame="x.n", columnID=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=2, frame="z", columnID=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(id=2, frame="z", columnID=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=3, frame="neg", columnID=100)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(id=3, frame="neg", columnID=100)`); err != nil {
t.Fatal(err)
}
// Set row attributes.
if _, err := m.Query("d", "", `SetRowAttrs(id=1, frame="x.n", x=100)`); err != nil {
if _, err := m.Query("i", "", `SetRowAttrs(id=1, frame="x.n", x=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="x.n", x=-200)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(id=2, frame="x.n", x=-200)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="z", x=300)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(id=2, frame="z", x=300)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetRowAttrs(id=3, frame="neg", x=-0.44)`); err != nil {
} else if _, err := m.Query("i", "", `SetRowAttrs(id=3, frame="neg", x=-0.44)`); err != nil {
t.Fatal(err)
}
// Query row x.n/1.
if res, err := m.Query("d", "", `Bitmap(id=1, frame="x.n")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x.n/2.
if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(id=2, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -160,19 +160,19 @@ func TestMain_SetRowAttrs(t *testing.T) {
}
// Query rows after reopening.
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=3, frame="neg")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=3, frame="neg")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x.n/2.
if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil {
if res, err := m.Query("i", "", `Bitmap(id=2, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -186,26 +186,26 @@ func TestMain_SetColumnAttrs(t *testing.T) {
// Create frames.
client := m.Client()
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Set bits on row.
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil {
if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=101)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=101)`); err != nil {
t.Fatal(err)
}
// Set column attributes.
if _, err := m.Query("d", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil {
if _, err := m.Query("i", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil {
t.Fatal(err)
}
// Query row.
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -216,7 +216,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
// Query row after reopening.
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
@ -230,26 +230,26 @@ func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) {
// Create frames.
client := m.Client()
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrDatabaseExists {
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Set bits on row.
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil {
if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil {
} else if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil {
t.Fatal(err)
}
// Set column attributes.
if _, err := m.Query("d", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil {
if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil {
t.Fatal(err)
}
// Query row.
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -274,14 +274,14 @@ func TestMain_FrameRestore(t *testing.T) {
// Create frames.
client := m0.Client()
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
if err := client.CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := client.CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil {
} else if err := client.CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Write data on first cluster.
if _, err := m0.Query("d", "", `
if _, err := m0.Query("x", "", `
SetBit(id=1, frame="f", columnID=100)
SetBit(id=1, frame="f", columnID=1000)
SetBit(id=1, frame="f", columnID=100000)
@ -294,7 +294,7 @@ func TestMain_FrameRestore(t *testing.T) {
}
// Query row on first cluster.
if res, err := m0.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil {
if res, err := m0.Query("x", "", `Bitmap(id=1, frame="f")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -308,16 +308,16 @@ func TestMain_FrameRestore(t *testing.T) {
client, err := pilosa.NewClient(m2.Server.Host)
if err != nil {
t.Fatal(err)
} else if err := m2.Client().CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
} else if err := m2.Client().CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal(err)
} else if err := m2.Client().CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil {
} else if err := m2.Client().CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "d", "f"); err != nil {
} else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "x", "f"); err != nil {
t.Fatal(err)
}
// Query row on second cluster.
if res, err := m2.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil {
if res, err := m2.Query("x", "", `Bitmap(id=1, frame="f")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
@ -431,8 +431,9 @@ func (m *Main) Client() *pilosa.Client {
}
// Query executes a query against the program through the HTTP API.
func (m *Main) Query(db, rawQuery, query string) (string, error) {
resp := MustDo("POST", m.URL()+fmt.Sprintf("/db/%s/query?", db)+rawQuery, query)
func (m *Main) Query(index, rawQuery, query string) (string, error) {
fmt.Println("Query:", index, query)
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query)
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
}

12
view.go
View file

@ -26,7 +26,7 @@ func IsValidView(name string) bool {
type View struct {
mu sync.Mutex
path string
db string
index string
frame string
name string
@ -43,10 +43,10 @@ type View struct {
}
// NewView returns a new instance of View.
func NewView(path, db, frame, name string, cacheSize uint32) *View {
func NewView(path, index, frame, name string, cacheSize uint32) *View {
return &View{
path: path,
db: db,
index: index,
frame: frame,
name: name,
cacheSize: cacheSize,
@ -62,8 +62,8 @@ func NewView(path, db, frame, name string, cacheSize uint32) *View {
// Name returns the name the view was initialized with.
func (v *View) Name() string { return v.name }
// DB returns the database name the view was initialized with.
func (v *View) DB() string { return v.db }
// Index returns the index name the view was initialized with.
func (v *View) Index() string { return v.index }
// Frame returns the frame name the view was initialized with.
func (v *View) Frame() string { return v.frame }
@ -216,7 +216,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
}
func (v *View) newFragment(path string, slice uint64) *Fragment {
frag := NewFragment(path, v.db, v.frame, v.name, slice)
frag := NewFragment(path, v.index, v.frame, v.name, slice)
frag.cacheType = v.cacheType
frag.cacheSize = v.cacheSize
frag.LogOutput = v.LogOutput

View file

@ -14,7 +14,7 @@ type View struct {
}
// NewView returns a new instance of View with a temporary path.
func NewView(db, frame, name string) *View {
func NewView(index, frame, name string) *View {
file, err := ioutil.TempFile("", "pilosa-view-")
if err != nil {
panic(err)
@ -22,7 +22,7 @@ func NewView(db, frame, name string) *View {
file.Close()
v := &View{
View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize),
View: pilosa.NewView(file.Name(), index, frame, name, pilosa.DefaultCacheSize),
RowAttrStore: MustOpenAttrStore(),
}
v.View.RowAttrStore = v.RowAttrStore.AttrStore
@ -30,8 +30,8 @@ func NewView(db, frame, name string) *View {
}
// MustOpenView creates and opens an view at a temporary path. Panic on error.
func MustOpenView(db, frame, name string) *View {
v := NewView(db, frame, name)
func MustOpenView(index, frame, name string) *View {
v := NewView(index, frame, name)
if err := v.Open(); err != nil {
panic(err)
}
@ -52,7 +52,7 @@ func (v *View) Reopen() error {
return err
}
v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize)
v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize)
v.View.RowAttrStore = v.RowAttrStore.AttrStore
if err := v.Open(); err != nil {
return err