mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-08-28 10:54:59 +00:00
fixed conflicts from the db index change
Conflicts: cluster.go executor_test.go handler_test.go server.go
This commit is contained in:
commit
e699c55700
63 changed files with 4200 additions and 3713 deletions
2
.gitignore
vendored
2
.gitignore
vendored
|
|
@ -2,3 +2,5 @@ default.etcd/
|
|||
*.test
|
||||
vendor
|
||||
.protoc-gen-gofast
|
||||
|
||||
/datadog/.DS_Store
|
||||
|
|
|
|||
4
NOTES
4
NOTES
|
|
@ -1,10 +1,10 @@
|
|||
|
||||
DB Profile
|
||||
Index Column
|
||||
┌───────────▼────────────────────────────┐
|
||||
│0000000000000000000000000000000000000000│
|
||||
│0000000000000000000000000000000000000000│
|
||||
│0000000000000000000000000000000000000000│
|
||||
Bitmap──▶0000000000000000000000000000000000000000│
|
||||
Row──▶0000000000000000000000000000000000000000│
|
||||
│0000000000000000000000000000000000000000│
|
||||
│────────────────────────────────────────┤
|
||||
│0000000000000000000000000000000000000000│
|
||||
|
|
|
|||
50
README.md
50
README.md
|
|
@ -1,6 +1,6 @@
|
|||
# pilosa
|
||||
|
||||
Pilosa is a bitmap index database.
|
||||
Pilosa is a bitmap index.
|
||||
|
||||
[](https://travis-ci.com/pilosa/pilosa)
|
||||
|
||||
|
|
@ -123,60 +123,60 @@ Return the version of Pilosa:
|
|||
$ curl "http://127.0.0.1:10101/version"
|
||||
```
|
||||
|
||||
Return a list of all databases and frames in the index:
|
||||
Return a list of all indexes and frames in the index:
|
||||
```sh
|
||||
$ curl "http://127.0.0.1:10101/schema"
|
||||
```
|
||||
|
||||
### Database and Frame Schema
|
||||
### Index and Frame Schema
|
||||
|
||||
Before running a query, the corresponding database and frame must be created. Note that database and frame names can contain only lower case letters, numbers, dash (`-`), underscore (`_`) and dot (`.`).
|
||||
Before running a query, the corresponding index and frame must be created. Note that index and frame names can contain only lower case letters, numbers, dash (`-`), underscore (`_`) and dot (`.`).
|
||||
|
||||
You can create the database `sample-db` using:
|
||||
You can create the index `sample-idx` using:
|
||||
|
||||
```sh
|
||||
$ curl -XPOST "http://127.0.0.1:10101/db" \
|
||||
-d '{"db": "sample-db"}'
|
||||
$ curl -XPOST "http://127.0.0.1:10101/index" \
|
||||
-d '{"index": "sample-idx"}'
|
||||
```
|
||||
|
||||
Optionally, you can specify the column label on database creation:
|
||||
Optionally, you can specify the column label on index creation:
|
||||
|
||||
```sh
|
||||
$ curl -XPOST "http://127.0.0.1:10101/db" \
|
||||
-d '{"db": "sample-db", "options": {"columnLabel": "user"}}'
|
||||
$ curl -XPOST "http://127.0.0.1:10101/index" \
|
||||
-d '{"index": "sample-idx", "options": {"columnLabel": "user"}}'
|
||||
```
|
||||
|
||||
The frame `collaboration` may be created using the following call:
|
||||
|
||||
```sh
|
||||
$ curl -XPOST "http://127.0.0.1:10101/frame" \
|
||||
-d '{"db": "sample-db", "frame": "collaboration"}'
|
||||
-d '{"index": "sample-idx", "frame": "collaboration"}'
|
||||
```
|
||||
|
||||
It is possible to specify the frame row label on frame creation:
|
||||
|
||||
```sh
|
||||
$ curl -XPOST "http://127.0.0.1:10101/frame" \
|
||||
-d '{"db": "sample-db", "frame": "collaboration", "options": {"rowLabel": "project"}}'
|
||||
-d '{"index": "sample-idx", "frame": "collaboration", "options": {"rowLabel": "project"}}'
|
||||
```
|
||||
|
||||
### Queries
|
||||
|
||||
Queries to Pilosa require sending a POST request where the query itself is sent as POST data.
|
||||
You specify the database on which to perform the query with a URL argument `db=database-name`.
|
||||
You specify the index on which to perform the query with a URL argument `index=index-name`.
|
||||
|
||||
In this section, we assume both the database `sample-db` with column label `user` and the frame `collaboration` with row label `project` was created.
|
||||
In this section, we assume both the index `sample-idx` with column label `user` and the frame `collaboration` with row label `project` was created.
|
||||
|
||||
A query sent to database `sample-db` will have the following format:
|
||||
A query sent to index `sample-idx` will have the following format:
|
||||
|
||||
```sh
|
||||
$ curl -X POST "http://127.0.0.1:10101/query?db=sample-db" -d 'Query()'
|
||||
$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'Query()'
|
||||
```
|
||||
|
||||
The `Query()` object referenced above should be made up of one or more of the query types listed below.
|
||||
So for example, a SetBit() query would look like this:
|
||||
```sh
|
||||
$ curl -X POST "http://127.0.0.1:10101/query?db=sample-db" -d 'SetBit(project=10, frame="collaboration", user=1)'
|
||||
$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'SetBit(project=10, frame="collaboration", user=1)'
|
||||
```
|
||||
|
||||
Query results have the format `{"results":[]}`, where `results` is a list of results for each `Query()`. This
|
||||
|
|
@ -184,7 +184,7 @@ means that you can provide multiple `Query()` objects with each HTTP request and
|
|||
the results of all of the queries.
|
||||
|
||||
```sh
|
||||
$ curl -X POST "http://127.0.0.1:10101/query?db=sample-db" -d 'Query() Query() Query()'
|
||||
$ curl -X POST "http://127.0.0.1:10101/query?index=sample-idx" -d 'Query() Query() Query()'
|
||||
```
|
||||
|
||||
---
|
||||
|
|
@ -209,17 +209,17 @@ A return value of `{"results":[true]}` indicates that the bit was toggled from 1
|
|||
A return value of `{"results":[false]}` indicates that the bit was already set to 0 and therefore nothing changed.
|
||||
|
||||
---
|
||||
#### SetBitmapAttrs()
|
||||
#### SetRowAttrs()
|
||||
```
|
||||
SetBitmapAttrs(project=10, frame="collaboration", stars=123, url="http://projects.pilosa.com/10", active=true)
|
||||
SetRowAttrs(project=10, frame="collaboration", stars=123, url="http://projects.pilosa.com/10", active=true)
|
||||
```
|
||||
Returns `{"results":[null]}`
|
||||
|
||||
---
|
||||
#### SetProfileAttrs()
|
||||
#### SetColumnAttrs()
|
||||
---
|
||||
```
|
||||
SetProfileAttrs(user=10, friends=123, username="mrpi", active=true)
|
||||
SetColumnAttrs(user=10, friends=123, username="mrpi", active=true)
|
||||
```
|
||||
|
||||
Returns `{"results":[null]}`
|
||||
|
|
@ -230,11 +230,11 @@ Returns `{"results":[null]}`
|
|||
Bitmap(project=10, frame="collaboration")
|
||||
```
|
||||
Returns `{"results":[{"attrs":{"stars":123, "url":"http://projects.pilosa.com/10", "active":true},"bits":[1,2]}]}` where `attrs` are the
|
||||
attributes set using `SetBitmapAttrs()` and `bits` are the bits set using `SetBit()`.
|
||||
attributes set using `SetRowAttrs()` and `bits` are the bits set using `SetBit()`.
|
||||
|
||||
In order to return profile attributes attached to the profiles of a bitmap, add `&profiles=true` to the query string. Sample response:
|
||||
In order to return column attributes attached to the columns of a bitmap, add `&columnAttrs=true` to the query string. Sample response:
|
||||
```
|
||||
{"results":[{"attrs":{},"bits":[10]}],"profiles":[{"user":10,"attrs":{"friends":123, "username":"mrpi", "active":true}}]}
|
||||
{"results":[{"attrs":{},"bits":[10]}],"columnAttrs":[{"user":10,"attrs":{"friends":123, "username":"mrpi", "active":true}}]}
|
||||
```
|
||||
|
||||
---
|
||||
|
|
|
|||
2
attr.go
2
attr.go
|
|
@ -226,7 +226,7 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro
|
|||
return m, nil
|
||||
}
|
||||
|
||||
// txAttrs returns a map of attributes for a bitmap.
|
||||
// txAttrs returns a map of attributes for an id.
|
||||
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
|
||||
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
|
||||
if v == nil {
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import (
|
|||
"github.com/pilosa/pilosa"
|
||||
)
|
||||
|
||||
// Ensure database can set and retrieve profile attributes.
|
||||
// Ensure database can set and retrieve column attributes.
|
||||
func TestAttrStore_Attrs(t *testing.T) {
|
||||
s := MustOpenAttrStore()
|
||||
defer s.Close()
|
||||
|
|
@ -23,14 +23,14 @@ func TestAttrStore_Attrs(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Retrieve attributes for profile #1.
|
||||
// Retrieve attributes for column #1.
|
||||
if m, err := s.Attrs(1); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) {
|
||||
t.Fatalf("unexpected attrs(1): %#v", m)
|
||||
}
|
||||
|
||||
// Retrieve attributes for profile #2.
|
||||
// Retrieve attributes for column #2.
|
||||
if m, err := s.Attrs(2); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) {
|
||||
|
|
|
|||
25
broadcast.go
25
broadcast.go
|
|
@ -34,6 +34,11 @@ func (s *StaticNodeSet) Open() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (s *StaticNodeSet) Join(nodes []*Node) error {
|
||||
s.nodes = nodes
|
||||
return nil
|
||||
}
|
||||
|
||||
// Broadcaster is an interface for broadcasting messages.
|
||||
type Broadcaster interface {
|
||||
SendSync(pb proto.Message) error
|
||||
|
|
@ -84,8 +89,8 @@ var NopBroadcastReceiver = &nopBroadcastReceiver{}
|
|||
|
||||
const (
|
||||
MessageTypeCreateSlice = 1
|
||||
MessageTypeCreateDB = 2
|
||||
MessageTypeDeleteDB = 3
|
||||
MessageTypeCreateIndex = 2
|
||||
MessageTypeDeleteIndex = 3
|
||||
MessageTypeCreateFrame = 4
|
||||
MessageTypeDeleteFrame = 5
|
||||
)
|
||||
|
|
@ -95,10 +100,10 @@ func MarshalMessage(m proto.Message) ([]byte, error) {
|
|||
switch obj := m.(type) {
|
||||
case *internal.CreateSliceMessage:
|
||||
typ = MessageTypeCreateSlice
|
||||
case *internal.CreateDBMessage:
|
||||
typ = MessageTypeCreateDB
|
||||
case *internal.DeleteDBMessage:
|
||||
typ = MessageTypeDeleteDB
|
||||
case *internal.CreateIndexMessage:
|
||||
typ = MessageTypeCreateIndex
|
||||
case *internal.DeleteIndexMessage:
|
||||
typ = MessageTypeDeleteIndex
|
||||
case *internal.CreateFrameMessage:
|
||||
typ = MessageTypeCreateFrame
|
||||
case *internal.DeleteFrameMessage:
|
||||
|
|
@ -120,10 +125,10 @@ func UnmarshalMessage(buf []byte) (proto.Message, error) {
|
|||
switch typ {
|
||||
case MessageTypeCreateSlice:
|
||||
m = &internal.CreateSliceMessage{}
|
||||
case MessageTypeCreateDB:
|
||||
m = &internal.CreateDBMessage{}
|
||||
case MessageTypeDeleteDB:
|
||||
m = &internal.DeleteDBMessage{}
|
||||
case MessageTypeCreateIndex:
|
||||
m = &internal.CreateIndexMessage{}
|
||||
case MessageTypeDeleteIndex:
|
||||
m = &internal.DeleteIndexMessage{}
|
||||
case MessageTypeCreateFrame:
|
||||
m = &internal.CreateFrameMessage{}
|
||||
case MessageTypeDeleteFrame:
|
||||
|
|
|
|||
91
broadcast_test.go
Normal file
91
broadcast_test.go
Normal file
|
|
@ -0,0 +1,91 @@
|
|||
package pilosa_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/pilosa/pilosa/internal"
|
||||
)
|
||||
|
||||
// Ensure a message can be marshaled and unmarshaled.
|
||||
func TestMessage_Marshal(t *testing.T) {
|
||||
|
||||
testMessageMarshal(t, &internal.CreateSliceMessage{
|
||||
Index: "i",
|
||||
Slice: 8,
|
||||
})
|
||||
|
||||
testMessageMarshal(t, &internal.DeleteIndexMessage{
|
||||
Index: "i",
|
||||
})
|
||||
}
|
||||
|
||||
func testMessageMarshal(t *testing.T, m proto.Message) {
|
||||
marshalled, err := pilosa.MarshalMessage(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unmarshalled, err := pilosa.UnmarshalMessage(marshalled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(unmarshalled, m) {
|
||||
t.Fatalf("unexpected message marshalling: %s", unmarshalled)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure that BroadcastReceiver can register a BroadcastHandler.
|
||||
func TestBroadcast_BroadcastReceiver(t *testing.T) {
|
||||
|
||||
s := pilosa.NewServer()
|
||||
|
||||
sbr := NewSimpleBroadcastReceiver()
|
||||
sbh := NewSimpleBroadcastHandler()
|
||||
|
||||
s.BroadcastReceiver = sbr
|
||||
s.BroadcastReceiver.Start(sbh)
|
||||
|
||||
msg := &internal.DeleteIndexMessage{
|
||||
Index: "i",
|
||||
}
|
||||
|
||||
s.BroadcastReceiver.(*SimpleBroadcastReceiver).Receive(msg)
|
||||
|
||||
// Make sure the message received is what was sentd
|
||||
if !reflect.DeepEqual(sbh.receivedMessage, msg) {
|
||||
t.Fatalf("unexpected message: %s", sbh.receivedMessage)
|
||||
}
|
||||
}
|
||||
|
||||
type SimpleBroadcastReceiver struct {
|
||||
broadcastHandler pilosa.BroadcastHandler
|
||||
}
|
||||
|
||||
func NewSimpleBroadcastReceiver() *SimpleBroadcastReceiver {
|
||||
return &SimpleBroadcastReceiver{}
|
||||
}
|
||||
|
||||
func (r *SimpleBroadcastReceiver) Start(h pilosa.BroadcastHandler) error {
|
||||
r.broadcastHandler = h
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *SimpleBroadcastReceiver) Receive(pb proto.Message) error {
|
||||
r.broadcastHandler.ReceiveMessage(pb)
|
||||
return nil
|
||||
}
|
||||
|
||||
type SimpleBroadcastHandler struct {
|
||||
receivedMessage proto.Message
|
||||
}
|
||||
|
||||
func NewSimpleBroadcastHandler() *SimpleBroadcastHandler {
|
||||
return &SimpleBroadcastHandler{}
|
||||
}
|
||||
|
||||
func (h *SimpleBroadcastHandler) ReceiveMessage(pb proto.Message) error {
|
||||
h.receivedMessage = pb.(proto.Message)
|
||||
return nil
|
||||
}
|
||||
64
cache.go
64
cache.go
|
|
@ -17,15 +17,15 @@ const (
|
|||
ThresholdFactor = 1.1
|
||||
)
|
||||
|
||||
// Cache represents a cache for bitmap counts.
|
||||
// Cache represents a cache of counts.
|
||||
type Cache interface {
|
||||
Add(bitmapID uint64, n uint64)
|
||||
BulkAdd(bitmapID uint64, n uint64)
|
||||
Get(bitmapID uint64) uint64
|
||||
Add(id uint64, n uint64)
|
||||
BulkAdd(id uint64, n uint64)
|
||||
Get(id uint64) uint64
|
||||
Len() int
|
||||
|
||||
// Returns a list of all bitmap IDs.
|
||||
BitmapIDs() []uint64
|
||||
// Returns a list of all IDs.
|
||||
IDs() []uint64
|
||||
|
||||
// Updates the cache, if necessary.
|
||||
Invalidate()
|
||||
|
|
@ -58,19 +58,19 @@ func NewLRUCache(maxEntries uint32) *LRUCache {
|
|||
return c
|
||||
}
|
||||
|
||||
func (c *LRUCache) BulkAdd(bitmapID, n uint64) {
|
||||
c.Add(bitmapID, n)
|
||||
func (c *LRUCache) BulkAdd(id, n uint64) {
|
||||
c.Add(id, n)
|
||||
}
|
||||
|
||||
// Add adds a bitmap to the cache.
|
||||
func (c *LRUCache) Add(bitmapID, n uint64) {
|
||||
c.cache.Add(bitmapID, n)
|
||||
c.counts[bitmapID] = n
|
||||
// Add adds a count to the cache.
|
||||
func (c *LRUCache) Add(id, n uint64) {
|
||||
c.cache.Add(id, n)
|
||||
c.counts[id] = n
|
||||
}
|
||||
|
||||
// Get returns a bitmap with a given id.
|
||||
func (c *LRUCache) Get(bitmapID uint64) uint64 {
|
||||
n, _ := c.cache.Get(bitmapID)
|
||||
// Get returns a count for a given id.
|
||||
func (c *LRUCache) Get(id uint64) uint64 {
|
||||
n, _ := c.cache.Get(id)
|
||||
nn, _ := n.(uint64)
|
||||
return nn
|
||||
}
|
||||
|
|
@ -88,8 +88,8 @@ func (c *LRUCache) Recalculate() {
|
|||
c.stats.Gauge("LRUCache", float64(c.cache.Len()))
|
||||
}
|
||||
|
||||
// BitmapIDs returns a list of all bitmap IDs in the cache.
|
||||
func (c *LRUCache) BitmapIDs() []uint64 {
|
||||
// IDs returns a list of all IDs in the cache.
|
||||
func (c *LRUCache) IDs() []uint64 {
|
||||
a := make([]uint64, 0, len(c.counts))
|
||||
for id := range c.counts {
|
||||
a = append(a, id)
|
||||
|
|
@ -153,36 +153,36 @@ func NewRankCache(maxEntries uint32) *RankCache {
|
|||
}
|
||||
}
|
||||
|
||||
// Add adds a bitmap to the cache.
|
||||
func (c *RankCache) Add(bitmapID uint64, n uint64) {
|
||||
// Add adds a count to the cache.
|
||||
func (c *RankCache) Add(id uint64, n uint64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
// Ignore if the bit count on the bitmap is below the threshold.
|
||||
// Ignore if the bit count is below the threshold.
|
||||
if n < c.thresholdValue {
|
||||
return
|
||||
}
|
||||
|
||||
c.entries[bitmapID] = n
|
||||
c.entries[id] = n
|
||||
|
||||
c.invalidate()
|
||||
}
|
||||
|
||||
// BulkAdd adds a bitmap to the cache unsorted. You should Invalidate after completion.
|
||||
func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) {
|
||||
// BulkAdd adds a count to the cache unsorted. You should Invalidate after completion.
|
||||
func (c *RankCache) BulkAdd(id uint64, n uint64) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if n < c.thresholdValue {
|
||||
return
|
||||
}
|
||||
|
||||
c.entries[bitmapID] = n
|
||||
c.entries[id] = n
|
||||
}
|
||||
|
||||
// Get returns a bitmap with a given id.
|
||||
func (c *RankCache) Get(bitmapID uint64) uint64 {
|
||||
// Get returns a count for a given id.
|
||||
func (c *RankCache) Get(id uint64) uint64 {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
return c.entries[bitmapID]
|
||||
return c.entries[id]
|
||||
}
|
||||
|
||||
// Len returns the number of items in the cache.
|
||||
|
|
@ -192,8 +192,8 @@ func (c *RankCache) Len() int {
|
|||
return len(c.entries)
|
||||
}
|
||||
|
||||
// BitmapIDs returns a list of all bitmap IDs in the cache.
|
||||
func (c *RankCache) BitmapIDs() []uint64 {
|
||||
// IDs returns a list of all IDs in the cache.
|
||||
func (c *RankCache) IDs() []uint64 {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
a := make([]uint64, 0, len(c.entries))
|
||||
|
|
@ -267,7 +267,7 @@ func (c *RankCache) SetStats(s StatsClient) {
|
|||
c.stats = s
|
||||
}
|
||||
|
||||
// Top returns an ordered list of bitmaps.
|
||||
// Top returns an ordered list of pairs.
|
||||
func (c *RankCache) Top() []BitmapPair { return c.rankings }
|
||||
|
||||
// WriteTo writes the cache to w.
|
||||
|
|
@ -283,7 +283,7 @@ func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
|
|||
// Ensure RankCache implements Cache.
|
||||
var _ Cache = &RankCache{}
|
||||
|
||||
// BitmapPair represents a bitmap with an associated identifier.
|
||||
// BitmapPair represents a id/count pair with an associated identifier.
|
||||
type BitmapPair struct {
|
||||
ID uint64
|
||||
Count uint64
|
||||
|
|
@ -296,7 +296,7 @@ func (p BitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
|||
func (p BitmapPairs) Len() int { return len(p) }
|
||||
func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
|
||||
|
||||
// Pair holds a bitmap id and its count.
|
||||
// Pair holds an id/count pair.
|
||||
type Pair struct {
|
||||
ID uint64 `json:"id"`
|
||||
Count uint64 `json:"count"`
|
||||
|
|
|
|||
202
client.go
202
client.go
|
|
@ -45,18 +45,18 @@ func NewClient(host string) (*Client, error) {
|
|||
// Host returns the host the client was initialized with.
|
||||
func (c *Client) Host() string { return c.host }
|
||||
|
||||
// MaxSliceByDatabase returns the number of slices on a server by database.
|
||||
func (c *Client) MaxSliceByDatabase(ctx context.Context) (map[string]uint64, error) {
|
||||
return c.maxSliceByDatabase(ctx, false)
|
||||
// MaxSliceByIndex returns the number of slices on a server by index.
|
||||
func (c *Client) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
|
||||
return c.maxSliceByIndex(ctx, false)
|
||||
}
|
||||
|
||||
// MaxInverseSliceByDatabase returns the number of inverse slices on a server by database.
|
||||
func (c *Client) MaxInverseSliceByDatabase(ctx context.Context) (map[string]uint64, error) {
|
||||
return c.maxSliceByDatabase(ctx, true)
|
||||
// MaxInverseSliceByIndex returns the number of inverse slices on a server by index.
|
||||
func (c *Client) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) {
|
||||
return c.maxSliceByIndex(ctx, true)
|
||||
}
|
||||
|
||||
// maxSliceByDatabase returns the number of slices on a server by database.
|
||||
func (c *Client) maxSliceByDatabase(ctx context.Context, inverse bool) (map[string]uint64, error) {
|
||||
// maxSliceByIndex returns the number of slices on a server by index.
|
||||
func (c *Client) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
|
||||
// Execute request against the host.
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
|
|
@ -92,8 +92,8 @@ func (c *Client) maxSliceByDatabase(ctx context.Context, inverse bool) (map[stri
|
|||
return rsp.MaxSlices, nil
|
||||
}
|
||||
|
||||
// Schema returns all database and frame schema information.
|
||||
func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) {
|
||||
// Schema returns all index and frame schema information.
|
||||
func (c *Client) Schema(ctx context.Context) ([]*IndexInfo, error) {
|
||||
// Execute request against the host.
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
|
|
@ -122,13 +122,13 @@ func (c *Client) Schema(ctx context.Context) ([]*DBInfo, error) {
|
|||
} else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
|
||||
return nil, fmt.Errorf("json decode: %s", err)
|
||||
}
|
||||
return rsp.DBs, nil
|
||||
return rsp.Indexes, nil
|
||||
}
|
||||
|
||||
// CreateDB creates a new database on the server.
|
||||
func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error {
|
||||
// CreateIndex creates a new index on the server.
|
||||
func (c *Client) CreateIndex(ctx context.Context, index string, opt IndexOptions) error {
|
||||
// Encode query request.
|
||||
buf, err := json.Marshal(&postDBRequest{
|
||||
buf, err := json.Marshal(&postIndexRequest{
|
||||
Options: opt,
|
||||
})
|
||||
if err != nil {
|
||||
|
|
@ -136,7 +136,7 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error {
|
|||
}
|
||||
|
||||
// Create URL & HTTP request.
|
||||
u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/db/%s", db)}
|
||||
u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/index/%s", index)}
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -164,20 +164,20 @@ func (c *Client) CreateDB(ctx context.Context, db string, opt DBOptions) error {
|
|||
case http.StatusOK:
|
||||
return nil // ok
|
||||
case http.StatusConflict:
|
||||
return ErrDatabaseExists
|
||||
return ErrIndexExists
|
||||
default:
|
||||
return errors.New(string(body))
|
||||
}
|
||||
}
|
||||
|
||||
// FragmentNodes returns a list of nodes that own a slice.
|
||||
func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([]*Node, error) {
|
||||
func (c *Client) FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) {
|
||||
// Execute request against the host.
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: c.host,
|
||||
Path: "/fragment/nodes",
|
||||
RawQuery: (url.Values{"db": {db}, "slice": {strconv.FormatUint(slice, 10)}}).Encode(),
|
||||
RawQuery: (url.Values{"index": {index}, "slice": {strconv.FormatUint(slice, 10)}}).Encode(),
|
||||
}
|
||||
|
||||
// Build request.
|
||||
|
|
@ -205,10 +205,10 @@ func (c *Client) FragmentNodes(ctx context.Context, db string, slice uint64) ([]
|
|||
return a, nil
|
||||
}
|
||||
|
||||
// ExecuteQuery executes query against db on the server.
|
||||
func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedirect bool) (result interface{}, err error) {
|
||||
if db == "" {
|
||||
return nil, ErrDatabaseRequired
|
||||
// ExecuteQuery executes query against index on the server.
|
||||
func (c *Client) ExecuteQuery(ctx context.Context, index, query string, allowRedirect bool) (result interface{}, err error) {
|
||||
if index == "" {
|
||||
return nil, ErrIndexRequired
|
||||
} else if query == "" {
|
||||
return nil, ErrQueryRequired
|
||||
}
|
||||
|
|
@ -226,7 +226,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire
|
|||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: c.host,
|
||||
Path: fmt.Sprintf("/db/%s/query", db),
|
||||
Path: fmt.Sprintf("/index/%s/query", index),
|
||||
}
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
|
|
@ -262,14 +262,14 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire
|
|||
return qresp, nil
|
||||
}
|
||||
|
||||
// ExecutePQL executes query string against db on the server.
|
||||
func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{}, error) {
|
||||
// ExecutePQL executes query string against index on the server.
|
||||
func (c *Client) ExecutePQL(ctx context.Context, index, query string) (interface{}, error) {
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: c.host,
|
||||
Path: "/query",
|
||||
RawQuery: url.Values{
|
||||
"db": {db},
|
||||
"index": {index},
|
||||
}.Encode(),
|
||||
}
|
||||
|
||||
|
|
@ -297,20 +297,20 @@ func (c *Client) ExecutePQL(ctx context.Context, db, query string) (interface{},
|
|||
}
|
||||
|
||||
// Import bulk imports bits for a single slice to a host.
|
||||
func (c *Client) Import(ctx context.Context, db, frame string, slice uint64, bits []Bit) error {
|
||||
if db == "" {
|
||||
return ErrDatabaseRequired
|
||||
func (c *Client) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
return ErrFrameRequired
|
||||
}
|
||||
|
||||
buf, err := MarshalImportPayload(db, frame, slice, bits)
|
||||
buf, err := MarshalImportPayload(index, frame, slice, bits)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Error Creating Payload: %s", err)
|
||||
}
|
||||
|
||||
// Retrieve a list of nodes that own the slice.
|
||||
nodes, err := c.FragmentNodes(ctx, db, slice)
|
||||
nodes, err := c.FragmentNodes(ctx, index, slice)
|
||||
if err != nil {
|
||||
return fmt.Errorf("slice nodes: %s", err)
|
||||
}
|
||||
|
|
@ -325,19 +325,19 @@ func (c *Client) Import(ctx context.Context, db, frame string, slice uint64, bit
|
|||
return nil
|
||||
}
|
||||
|
||||
func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, error) {
|
||||
// Separate bitmap and profile IDs to reduce allocations.
|
||||
bitmapIDs := Bits(bits).BitmapIDs()
|
||||
profileIDs := Bits(bits).ProfileIDs()
|
||||
func MarshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) {
|
||||
// Separate row and column IDs to reduce allocations.
|
||||
rowIDs := Bits(bits).RowIDs()
|
||||
columnIDs := Bits(bits).ColumnIDs()
|
||||
timestamps := Bits(bits).Timestamps()
|
||||
|
||||
// Marshal bits to protobufs.
|
||||
buf, err := proto.Marshal(&internal.ImportRequest{
|
||||
DB: db,
|
||||
Index: index,
|
||||
Frame: frame,
|
||||
Slice: slice,
|
||||
BitmapIDs: bitmapIDs,
|
||||
ProfileIDs: profileIDs,
|
||||
RowIDs: rowIDs,
|
||||
ColumnIDs: columnIDs,
|
||||
Timestamps: timestamps,
|
||||
})
|
||||
if err != nil {
|
||||
|
|
@ -385,15 +385,15 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error {
|
|||
}
|
||||
|
||||
// ExportCSV bulk exports data for a single slice from a host to CSV format.
|
||||
func (c *Client) ExportCSV(ctx context.Context, db, frame string, slice uint64, w io.Writer) error {
|
||||
if db == "" {
|
||||
return ErrDatabaseRequired
|
||||
func (c *Client) ExportCSV(ctx context.Context, index, frame string, slice uint64, w io.Writer) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
return ErrFrameRequired
|
||||
}
|
||||
|
||||
// Retrieve a list of nodes that own the slice.
|
||||
nodes, err := c.FragmentNodes(ctx, db, slice)
|
||||
nodes, err := c.FragmentNodes(ctx, index, slice)
|
||||
if err != nil {
|
||||
return fmt.Errorf("slice nodes: %s", err)
|
||||
}
|
||||
|
|
@ -403,7 +403,7 @@ func (c *Client) ExportCSV(ctx context.Context, db, frame string, slice uint64,
|
|||
for _, i := range rand.Perm(len(nodes)) {
|
||||
node := nodes[i]
|
||||
|
||||
if err := c.exportNodeCSV(ctx, node, db, frame, slice, w); err != nil {
|
||||
if err := c.exportNodeCSV(ctx, node, index, frame, slice, w); err != nil {
|
||||
e = fmt.Errorf("export node: host=%s, err=%s", node.Host, err)
|
||||
continue
|
||||
} else {
|
||||
|
|
@ -415,14 +415,14 @@ func (c *Client) ExportCSV(ctx context.Context, db, frame string, slice uint64,
|
|||
}
|
||||
|
||||
// exportNode copies a CSV export from a node to w.
|
||||
func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string, slice uint64, w io.Writer) error {
|
||||
func (c *Client) exportNodeCSV(ctx context.Context, node *Node, index, frame string, slice uint64, w io.Writer) error {
|
||||
// Create URL.
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: node.Host,
|
||||
Path: "/export",
|
||||
RawQuery: url.Values{
|
||||
"db": {db},
|
||||
"index": {index},
|
||||
"frame": {frame},
|
||||
"slice": {strconv.FormatUint(slice, 10)},
|
||||
}.Encode(),
|
||||
|
|
@ -457,9 +457,9 @@ func (c *Client) exportNodeCSV(ctx context.Context, node *Node, db, frame string
|
|||
}
|
||||
|
||||
// BackupTo backs up an entire frame from a cluster to w.
|
||||
func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view string) error {
|
||||
if db == "" {
|
||||
return ErrDatabaseRequired
|
||||
func (c *Client) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
return ErrFrameRequired
|
||||
}
|
||||
|
|
@ -468,14 +468,14 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view stri
|
|||
tw := tar.NewWriter(w)
|
||||
|
||||
// Find the maximum number of slices.
|
||||
maxSlices, err := c.MaxSliceByDatabase(ctx)
|
||||
maxSlices, err := c.MaxSliceByIndex(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("slice n: %s", err)
|
||||
}
|
||||
|
||||
// Backup every slice to the tar file.
|
||||
for i := uint64(0); i <= maxSlices[db]; i++ {
|
||||
if err := c.backupSliceTo(ctx, tw, db, frame, view, i); err != nil {
|
||||
for i := uint64(0); i <= maxSlices[index]; i++ {
|
||||
if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
@ -489,9 +489,9 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame, view stri
|
|||
}
|
||||
|
||||
// backupSliceTo backs up a single slice to tw.
|
||||
func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame, view string, slice uint64) error {
|
||||
func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error {
|
||||
// Return error if unable to backup from any slice.
|
||||
r, err := c.BackupSlice(ctx, db, frame, view, slice)
|
||||
r, err := c.BackupSlice(ctx, index, frame, view, slice)
|
||||
if err != nil {
|
||||
return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err)
|
||||
} else if r == nil {
|
||||
|
|
@ -527,16 +527,16 @@ func (c *Client) backupSliceTo(ctx context.Context, tw *tar.Writer, db, frame, v
|
|||
|
||||
// BackupSlice retrieves a streaming backup from a single slice.
|
||||
// This function tries slice owners until one succeeds.
|
||||
func (c *Client) BackupSlice(ctx context.Context, db, frame, view string, slice uint64) (io.ReadCloser, error) {
|
||||
func (c *Client) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) {
|
||||
// Retrieve a list of nodes that own the slice.
|
||||
nodes, err := c.FragmentNodes(ctx, db, slice)
|
||||
nodes, err := c.FragmentNodes(ctx, index, slice)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("slice nodes: %s", err)
|
||||
}
|
||||
|
||||
// Try to backup slice from each one until successful.
|
||||
for _, i := range rand.Perm(len(nodes)) {
|
||||
r, err := c.backupSliceNode(ctx, db, frame, view, slice, nodes[i])
|
||||
r, err := c.backupSliceNode(ctx, index, frame, view, slice, nodes[i])
|
||||
if err == nil {
|
||||
return r, nil // successfully attached
|
||||
} else if err == ErrFragmentNotFound {
|
||||
|
|
@ -550,13 +550,13 @@ func (c *Client) BackupSlice(ctx context.Context, db, frame, view string, slice
|
|||
return nil, fmt.Errorf("unable to connect to any owner")
|
||||
}
|
||||
|
||||
func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
|
||||
func (c *Client) backupSliceNode(ctx context.Context, index, frame, view string, slice uint64, node *Node) (io.ReadCloser, error) {
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: node.Host,
|
||||
Path: "/fragment/data",
|
||||
RawQuery: url.Values{
|
||||
"db": {db},
|
||||
"index": {index},
|
||||
"frame": {frame},
|
||||
"view": {view},
|
||||
"slice": {strconv.FormatUint(slice, 10)},
|
||||
|
|
@ -590,9 +590,9 @@ func (c *Client) backupSliceNode(ctx context.Context, db, frame, view string, sl
|
|||
}
|
||||
|
||||
// RestoreFrom restores a frame from a backup file to an entire cluster.
|
||||
func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame, view string) error {
|
||||
if db == "" {
|
||||
return ErrDatabaseRequired
|
||||
func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
} else if frame == "" {
|
||||
return ErrFrameRequired
|
||||
}
|
||||
|
|
@ -622,16 +622,16 @@ func (c *Client) RestoreFrom(ctx context.Context, r io.Reader, db, frame, view s
|
|||
}
|
||||
|
||||
// Restore file to all nodes that own it.
|
||||
if err := c.restoreSliceFrom(ctx, buf.Bytes(), db, frame, view, slice); err != nil {
|
||||
if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// restoreSliceFrom restores a single slice to all owning nodes.
|
||||
func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, view string, slice uint64) error {
|
||||
func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error {
|
||||
// Retrieve a list of nodes that own the slice.
|
||||
nodes, err := c.FragmentNodes(ctx, db, slice)
|
||||
nodes, err := c.FragmentNodes(ctx, index, slice)
|
||||
if err != nil {
|
||||
return fmt.Errorf("slice nodes: %s", err)
|
||||
}
|
||||
|
|
@ -643,7 +643,7 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, vi
|
|||
Host: node.Host,
|
||||
Path: "/fragment/data",
|
||||
RawQuery: url.Values{
|
||||
"db": {db},
|
||||
"index": {index},
|
||||
"frame": {frame},
|
||||
"view": {view},
|
||||
"slice": {strconv.FormatUint(slice, 10)},
|
||||
|
|
@ -674,9 +674,9 @@ func (c *Client) restoreSliceFrom(ctx context.Context, buf []byte, db, frame, vi
|
|||
}
|
||||
|
||||
// CreateFrame creates a new frame on the server.
|
||||
func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOptions) error {
|
||||
if db == "" {
|
||||
return ErrDatabaseRequired
|
||||
func (c *Client) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error {
|
||||
if index == "" {
|
||||
return ErrIndexRequired
|
||||
}
|
||||
|
||||
// Encode query request.
|
||||
|
|
@ -688,7 +688,7 @@ func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOpt
|
|||
}
|
||||
|
||||
// Create URL & HTTP request.
|
||||
u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/db/%s/frame/%s", db, frame)}
|
||||
u := url.URL{Scheme: "http", Host: c.host, Path: fmt.Sprintf("/index/%s/frame/%s", index, frame)}
|
||||
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -723,11 +723,11 @@ func (c *Client) CreateFrame(ctx context.Context, db, frame string, opt FrameOpt
|
|||
}
|
||||
|
||||
// RestoreFrame restores an entire frame from a host in another cluster.
|
||||
func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error {
|
||||
func (c *Client) RestoreFrame(ctx context.Context, host, index, frame string) error {
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: c.Host(),
|
||||
Path: fmt.Sprintf("/db/%s/frame/%s/restore", db, frame),
|
||||
Path: fmt.Sprintf("/index/%s/frame/%s/restore", index, frame),
|
||||
RawQuery: url.Values{
|
||||
"host": {host},
|
||||
}.Encode(),
|
||||
|
|
@ -757,12 +757,12 @@ func (c *Client) RestoreFrame(ctx context.Context, host, db, frame string) error
|
|||
}
|
||||
|
||||
// FrameViews returns a list of view names for a frame.
|
||||
func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, error) {
|
||||
func (c *Client) FrameViews(ctx context.Context, index, frame string) ([]string, error) {
|
||||
// Create URL & HTTP request.
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: c.host,
|
||||
Path: fmt.Sprintf("/db/%s/frame/%s/views", db, frame),
|
||||
Path: fmt.Sprintf("/index/%s/frame/%s/views", index, frame),
|
||||
}
|
||||
req, err := http.NewRequest("GET", u.String(), nil)
|
||||
if err != nil {
|
||||
|
|
@ -798,13 +798,13 @@ func (c *Client) FrameViews(ctx context.Context, db, frame string) ([]string, er
|
|||
|
||||
// FragmentBlocks returns a list of block checksums for a fragment on a host.
|
||||
// Only returns blocks which contain data.
|
||||
func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, slice uint64) ([]FragmentBlock, error) {
|
||||
func (c *Client) FragmentBlocks(ctx context.Context, index, frame, view string, slice uint64) ([]FragmentBlock, error) {
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: c.host,
|
||||
Path: "/fragment/blocks",
|
||||
RawQuery: url.Values{
|
||||
"db": {db},
|
||||
"index": {index},
|
||||
"frame": {frame},
|
||||
"view": {view},
|
||||
"slice": {strconv.FormatUint(slice, 10)},
|
||||
|
|
@ -843,10 +843,10 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, sli
|
|||
return rsp.Blocks, nil
|
||||
}
|
||||
|
||||
// BlockData returns bitmap/profile id pairs for a block.
|
||||
func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) {
|
||||
// BlockData returns row/column id pairs for a block.
|
||||
func (c *Client) BlockData(ctx context.Context, index, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) {
|
||||
buf, err := proto.Marshal(&internal.BlockDataRequest{
|
||||
DB: db,
|
||||
Index: index,
|
||||
Frame: frame,
|
||||
View: view,
|
||||
Slice: slice,
|
||||
|
|
@ -888,19 +888,19 @@ func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice ui
|
|||
} else if err := proto.Unmarshal(body, &rsp); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return rsp.BitmapIDs, rsp.ProfileIDs, nil
|
||||
return rsp.RowIDs, rsp.ColumnIDs, nil
|
||||
}
|
||||
|
||||
// ProfileAttrDiff returns data from differing blocks on a remote host.
|
||||
func (c *Client) ProfileAttrDiff(ctx context.Context, db string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
|
||||
// ColumnAttrDiff returns data from differing blocks on a remote host.
|
||||
func (c *Client) ColumnAttrDiff(ctx context.Context, index string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: c.host,
|
||||
Path: fmt.Sprintf("/db/%s/attr/diff", db),
|
||||
Path: fmt.Sprintf("/index/%s/attr/diff", index),
|
||||
}
|
||||
|
||||
// Encode request.
|
||||
buf, err := json.Marshal(postDBAttrDiffRequest{Blocks: blks})
|
||||
buf, err := json.Marshal(postIndexAttrDiffRequest{Blocks: blks})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -928,19 +928,19 @@ func (c *Client) ProfileAttrDiff(ctx context.Context, db string, blks []AttrBloc
|
|||
}
|
||||
|
||||
// Decode response object.
|
||||
var rsp postDBAttrDiffResponse
|
||||
var rsp postIndexAttrDiffResponse
|
||||
if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rsp.Attrs, nil
|
||||
}
|
||||
|
||||
// BitmapAttrDiff returns data from differing blocks on a remote host.
|
||||
func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
|
||||
// RowAttrDiff returns data from differing blocks on a remote host.
|
||||
func (c *Client) RowAttrDiff(ctx context.Context, index, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
|
||||
u := url.URL{
|
||||
Scheme: "http",
|
||||
Host: c.host,
|
||||
Path: fmt.Sprintf("/db/%s/frame/%s/attr/diff", db, frame),
|
||||
Path: fmt.Sprintf("/index/%s/frame/%s/attr/diff", index, frame),
|
||||
}
|
||||
|
||||
// Encode request.
|
||||
|
|
@ -983,8 +983,8 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At
|
|||
|
||||
// Bit represents the location of a single bit.
|
||||
type Bit struct {
|
||||
BitmapID uint64
|
||||
ProfileID uint64
|
||||
RowID uint64
|
||||
ColumnID uint64
|
||||
Timestamp int64
|
||||
}
|
||||
|
||||
|
|
@ -995,29 +995,29 @@ func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
|||
func (p Bits) Len() int { return len(p) }
|
||||
|
||||
func (p Bits) Less(i, j int) bool {
|
||||
if p[i].BitmapID == p[j].BitmapID {
|
||||
if p[i].ProfileID < p[j].ProfileID {
|
||||
if p[i].RowID == p[j].RowID {
|
||||
if p[i].ColumnID < p[j].ColumnID {
|
||||
return p[i].Timestamp < p[j].Timestamp
|
||||
}
|
||||
return p[i].ProfileID < p[j].ProfileID
|
||||
return p[i].ColumnID < p[j].ColumnID
|
||||
}
|
||||
return p[i].BitmapID < p[j].BitmapID
|
||||
return p[i].RowID < p[j].RowID
|
||||
}
|
||||
|
||||
// BitmapIDs returns a slice of all the bitmap IDs.
|
||||
func (a Bits) BitmapIDs() []uint64 {
|
||||
// RowIDs returns a slice of all the row IDs.
|
||||
func (a Bits) RowIDs() []uint64 {
|
||||
other := make([]uint64, len(a))
|
||||
for i := range a {
|
||||
other[i] = a[i].BitmapID
|
||||
other[i] = a[i].RowID
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
||||
// ProfileIDs returns a slice of all the profile IDs.
|
||||
func (a Bits) ProfileIDs() []uint64 {
|
||||
// ColumnIDs returns a slice of all the column IDs.
|
||||
func (a Bits) ColumnIDs() []uint64 {
|
||||
other := make([]uint64, len(a))
|
||||
for i := range a {
|
||||
other[i] = a[i].ProfileID
|
||||
other[i] = a[i].ColumnID
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
|
@ -1035,7 +1035,7 @@ func (a Bits) Timestamps() []int64 {
|
|||
func (a Bits) GroupBySlice() map[uint64][]Bit {
|
||||
m := make(map[uint64][]Bit)
|
||||
for _, bit := range a {
|
||||
slice := bit.ProfileID / SliceWidth
|
||||
slice := bit.ColumnID / SliceWidth
|
||||
m[slice] = append(m[slice], bit)
|
||||
}
|
||||
|
||||
|
|
@ -1053,7 +1053,7 @@ type BitsByPos []Bit
|
|||
func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||
func (p BitsByPos) Len() int { return len(p) }
|
||||
func (p BitsByPos) Less(i, j int) bool {
|
||||
p0, p1 := Pos(p[i].BitmapID, p[i].ProfileID), Pos(p[j].BitmapID, p[j].ProfileID)
|
||||
p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID)
|
||||
if p0 == p1 {
|
||||
return p[i].Timestamp < p[j].Timestamp
|
||||
}
|
||||
|
|
|
|||
213
client_test.go
213
client_test.go
|
|
@ -13,81 +13,81 @@ import (
|
|||
"github.com/pilosa/pilosa/pql"
|
||||
)
|
||||
|
||||
func createCluster(c *pilosa.Cluster) ([]*Server, []*Index) {
|
||||
func createCluster(c *pilosa.Cluster) ([]*Server, []*Holder) {
|
||||
numNodes := len(c.Nodes)
|
||||
idx := make([]*Index, numNodes)
|
||||
hldr := make([]*Holder, numNodes)
|
||||
server := make([]*Server, numNodes)
|
||||
for i := 0; i < numNodes; i++ {
|
||||
idx[i] = MustOpenIndex()
|
||||
hldr[i] = MustOpenHolder()
|
||||
server[i] = NewServer()
|
||||
server[i].Handler.Host = server[i].Host()
|
||||
server[i].Handler.Cluster = c
|
||||
server[i].Handler.Cluster.Nodes[i].Host = server[i].Host()
|
||||
server[i].Handler.Index = idx[i].Index
|
||||
server[i].Handler.Holder = hldr[i].Holder
|
||||
}
|
||||
return server, idx
|
||||
return server, hldr
|
||||
}
|
||||
|
||||
// Test distributed TopN Bitmap count across 3 nodes.
|
||||
// Test distributed TopN Row count across 3 nodes.
|
||||
func TestClient_MultiNode(t *testing.T) {
|
||||
cluster := NewCluster(3)
|
||||
s, idx := createCluster(cluster)
|
||||
s, hldr := createCluster(cluster)
|
||||
|
||||
for i := 0; i < len(cluster.Nodes); i++ {
|
||||
defer idx[i].Close()
|
||||
defer hldr[i].Close()
|
||||
defer s[i].Close()
|
||||
}
|
||||
|
||||
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
s[0].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
e := pilosa.NewExecutor()
|
||||
e.Index = idx[0].Index
|
||||
e.Holder = hldr[0].Holder
|
||||
e.Host = cluster.Nodes[0].Host
|
||||
e.Cluster = cluster
|
||||
return e.Execute(ctx, db, query, slices, opt)
|
||||
return e.Execute(ctx, index, query, slices, opt)
|
||||
}
|
||||
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
s[1].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
e := pilosa.NewExecutor()
|
||||
e.Index = idx[1].Index
|
||||
e.Holder = hldr[1].Holder
|
||||
e.Host = cluster.Nodes[1].Host
|
||||
e.Cluster = cluster
|
||||
return e.Execute(ctx, db, query, slices, opt)
|
||||
return e.Execute(ctx, index, query, slices, opt)
|
||||
}
|
||||
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
s[2].Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
e := pilosa.NewExecutor()
|
||||
e.Index = idx[2].Index
|
||||
e.Holder = hldr[2].Holder
|
||||
e.Host = cluster.Nodes[2].Host
|
||||
e.Cluster = cluster
|
||||
return e.Execute(ctx, db, query, slices, opt)
|
||||
return e.Execute(ctx, index, query, slices, opt)
|
||||
}
|
||||
|
||||
// Create a dispersed set of bitmaps across 3 nodes such that each individual node and slice width increment would reveal a different TopN.
|
||||
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4)
|
||||
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
|
||||
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
|
||||
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(1, 4)
|
||||
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(100, (SliceWidth*9)+10)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(4, (SliceWidth*9)+10, (SliceWidth*9)+11, (SliceWidth*9)+12, (SliceWidth*9)+13, (SliceWidth*9)+14, (SliceWidth*9)+15)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(2, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(3, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+3, (SliceWidth*9)+4, (SliceWidth*9)+5)
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 9).MustSetBits(22, (SliceWidth*9)+1, (SliceWidth*9)+2, (SliceWidth*9)+10)
|
||||
|
||||
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(100, (SliceWidth*10)+10)
|
||||
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12)
|
||||
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(4, (SliceWidth*10)+10, (SliceWidth*10)+11, (SliceWidth*10)+12, (SliceWidth*10)+13, (SliceWidth*10)+14, (SliceWidth*10)+15)
|
||||
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(2, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4)
|
||||
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(3, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+3, (SliceWidth*10)+4, (SliceWidth*10)+5)
|
||||
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).MustSetBits(22, (SliceWidth*10)+1, (SliceWidth*10)+2, (SliceWidth*10)+10)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(99, 1, 2, 3, 4)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(98, 1, 2, 3, 4, 5, 6)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(1, 4)
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(22, 1, 2, 3, 4, 5)
|
||||
|
||||
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(24, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13, (SliceWidth*6)+14)
|
||||
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
|
||||
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10)
|
||||
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10)
|
||||
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
|
||||
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
|
||||
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(20, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12, (SliceWidth*6)+13)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(21, (SliceWidth*6)+10)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(100, (SliceWidth*6)+10)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(99, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(98, (SliceWidth*6)+10, (SliceWidth*6)+11)
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).MustSetBits(22, (SliceWidth*6)+10, (SliceWidth*6)+11, (SliceWidth*6)+12)
|
||||
|
||||
// Rebuild the RankCache.
|
||||
// We have to do this to avoid the 10-second cache invalidation delay
|
||||
// built into cache.Invalidate()
|
||||
idx[0].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 0).RecalculateCache()
|
||||
idx[1].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 10).RecalculateCache()
|
||||
idx[2].MustCreateFragmentIfNotExists("d", "f.n", pilosa.ViewStandard, 6).RecalculateCache()
|
||||
hldr[0].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
|
||||
hldr[1].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 10).RecalculateCache()
|
||||
hldr[2].MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 6).RecalculateCache()
|
||||
|
||||
// Connect to each node to compare results.
|
||||
client := make([]*Client, 3)
|
||||
|
|
@ -96,9 +96,9 @@ func TestClient_MultiNode(t *testing.T) {
|
|||
client[2] = MustNewClient(s[0].Host())
|
||||
|
||||
topN := 4
|
||||
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f.n", topN)
|
||||
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN)
|
||||
|
||||
result, err := client[0].ExecuteQuery(context.Background(), "d", q, true)
|
||||
result, err := client[0].ExecuteQuery(context.Background(), "i", q, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -106,17 +106,17 @@ func TestClient_MultiNode(t *testing.T) {
|
|||
// Check the results before every node has the correct max slice value.
|
||||
pairs := result.(internal.QueryResponse).Results[0].Pairs
|
||||
for _, pair := range pairs {
|
||||
if pair.Key == 22 && pair.Count != 5 {
|
||||
if pair.Key == 22 && pair.Count != 11 {
|
||||
t.Fatalf("Invalid Cluster wide MaxSlice prevents accurate calculation of %s", pair)
|
||||
}
|
||||
}
|
||||
|
||||
// Set max slice to correct value.
|
||||
idx[0].DB("d").SetRemoteMaxSlice(10)
|
||||
idx[1].DB("d").SetRemoteMaxSlice(10)
|
||||
idx[2].DB("d").SetRemoteMaxSlice(10)
|
||||
hldr[0].Index("i").SetRemoteMaxSlice(10)
|
||||
hldr[1].Index("i").SetRemoteMaxSlice(10)
|
||||
hldr[2].Index("i").SetRemoteMaxSlice(10)
|
||||
|
||||
result, err = client[0].ExecuteQuery(context.Background(), "d", q, true)
|
||||
result, err = client[0].ExecuteQuery(context.Background(), "i", q, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -136,11 +136,11 @@ func TestClient_MultiNode(t *testing.T) {
|
|||
t.Fatalf("Invalid TopN result set: %s", spew.Sdump(result))
|
||||
}
|
||||
|
||||
result1, err := client[1].ExecuteQuery(context.Background(), "d", q, true)
|
||||
result1, err := client[1].ExecuteQuery(context.Background(), "i", q, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
result2, err := client[2].ExecuteQuery(context.Background(), "d", q, true)
|
||||
result2, err := client[2].ExecuteQuery(context.Background(), "i", q, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -157,66 +157,121 @@ func TestClient_MultiNode(t *testing.T) {
|
|||
|
||||
// Ensure client can bulk import data.
|
||||
func TestClient_Import(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Load bitmap into cache to ensure cache gets updated.
|
||||
f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
|
||||
f.Bitmap(0)
|
||||
f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
|
||||
f.Row(0)
|
||||
|
||||
s := NewServer()
|
||||
defer s.Close()
|
||||
s.Handler.Host = s.Host()
|
||||
s.Handler.Cluster = NewCluster(1)
|
||||
s.Handler.Cluster.Nodes[0].Host = s.Host()
|
||||
s.Handler.Index = idx.Index
|
||||
s.Handler.Holder = hldr.Holder
|
||||
|
||||
// Send import request.
|
||||
c := MustNewClient(s.Host())
|
||||
if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{
|
||||
{BitmapID: 0, ProfileID: 1},
|
||||
{BitmapID: 0, ProfileID: 5},
|
||||
{BitmapID: 200, ProfileID: 6},
|
||||
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
|
||||
{RowID: 0, ColumnID: 1},
|
||||
{RowID: 0, ColumnID: 5},
|
||||
{RowID: 200, ColumnID: 6},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data.
|
||||
if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) {
|
||||
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) {
|
||||
t.Fatalf("unexpected bits: %+v", a)
|
||||
}
|
||||
if a := f.Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) {
|
||||
if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) {
|
||||
t.Fatalf("unexpected bits: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure client can bulk import data to an inverse frame.
|
||||
func TestClient_ImportInverseEnabled(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
frameOpts := pilosa.FrameOptions{
|
||||
InverseEnabled: true,
|
||||
}
|
||||
frame, err := idx.CreateFrameIfNotExists("f", frameOpts)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
f, err := v.CreateFragmentIfNotExists(0)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// Load bitmap into cache to ensure cache gets updated.
|
||||
f.Row(0)
|
||||
|
||||
s := NewServer()
|
||||
defer s.Close()
|
||||
s.Handler.Host = s.Host()
|
||||
s.Handler.Cluster = NewCluster(1)
|
||||
s.Handler.Cluster.Nodes[0].Host = s.Host()
|
||||
s.Handler.Holder = hldr.Holder
|
||||
|
||||
// Send import request.
|
||||
c := MustNewClient(s.Host())
|
||||
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
|
||||
{RowID: 0, ColumnID: 1},
|
||||
{RowID: 0, ColumnID: 5},
|
||||
{RowID: 200, ColumnID: 5},
|
||||
{RowID: 200, ColumnID: 6},
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data.
|
||||
if a := f.Row(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) {
|
||||
t.Fatalf("unexpected bits: %+v", a)
|
||||
}
|
||||
if a := f.Row(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) {
|
||||
t.Fatalf("unexpected bits: %+v", a)
|
||||
}
|
||||
if a := f.Row(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) {
|
||||
t.Fatalf("unexpected bits: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure client backup and restore a frame.
|
||||
func TestClient_BackupRestore(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
|
||||
|
||||
s := NewServer()
|
||||
defer s.Close()
|
||||
s.Handler.Host = s.Host()
|
||||
s.Handler.Cluster = NewCluster(1)
|
||||
s.Handler.Cluster.Nodes[0].Host = s.Host()
|
||||
s.Handler.Index = idx.Index
|
||||
s.Handler.Holder = hldr.Holder
|
||||
|
||||
c := MustNewClient(s.Host())
|
||||
|
||||
// Backup from frame.
|
||||
var buf bytes.Buffer
|
||||
if err := c.BackupTo(context.Background(), &buf, "d", "f", pilosa.ViewStandard); err != nil {
|
||||
if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewStandard); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Restore to a different frame.
|
||||
if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
|
||||
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil {
|
||||
|
|
@ -224,42 +279,42 @@ func TestClient_BackupRestore(t *testing.T) {
|
|||
}
|
||||
|
||||
// Verify data.
|
||||
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
|
||||
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
|
||||
t.Fatalf("unexpected bits(0): %+v", a)
|
||||
}
|
||||
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
|
||||
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
|
||||
t.Fatalf("unexpected bits(0): %+v", a)
|
||||
}
|
||||
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
|
||||
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
|
||||
t.Fatalf("unexpected bits(0): %+v", a)
|
||||
}
|
||||
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) {
|
||||
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) {
|
||||
t.Fatalf("unexpected bits: %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
|
||||
func TestClient_FragmentBlocks(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Set two bits on blocks 0 & 3.
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100)
|
||||
|
||||
// Set a bit on a different slice.
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
|
||||
|
||||
s := NewServer()
|
||||
defer s.Close()
|
||||
s.Handler.Host = s.Host()
|
||||
s.Handler.Cluster = NewCluster(1)
|
||||
s.Handler.Cluster.Nodes[0].Host = s.Host()
|
||||
s.Handler.Index = idx.Index
|
||||
s.Handler.Holder = hldr.Holder
|
||||
|
||||
// Retrieve blocks.
|
||||
c := MustNewClient(s.Host())
|
||||
blocks, err := c.FragmentBlocks(context.Background(), "d", "f", pilosa.ViewStandard, 0)
|
||||
blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if len(blocks) != 2 {
|
||||
|
|
@ -271,7 +326,7 @@ func TestClient_FragmentBlocks(t *testing.T) {
|
|||
}
|
||||
|
||||
// Verify data matches local blocks.
|
||||
if a := idx.Fragment("d", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) {
|
||||
if a := hldr.Fragment("i", "f", pilosa.ViewStandard, 0).Blocks(); !reflect.DeepEqual(a, blocks) {
|
||||
t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
|
||||
}
|
||||
}
|
||||
|
|
|
|||
78
cluster.go
78
cluster.go
|
|
@ -4,6 +4,8 @@ import (
|
|||
"encoding/binary"
|
||||
"hash/fnv"
|
||||
"time"
|
||||
|
||||
"github.com/pilosa/pilosa/internal"
|
||||
)
|
||||
|
||||
const (
|
||||
|
|
@ -13,15 +15,30 @@ const (
|
|||
// DefaultReplicaN is the default number of replicas per partition.
|
||||
DefaultReplicaN = 1
|
||||
|
||||
// HealthStatus is the return value of the /health endpoint for a node in the cluster.
|
||||
HealthStatusUp = "UP"
|
||||
HealthStatusDown = "DOWN"
|
||||
// NodeState represents node state returned in /status endpoint for a node in the cluster.
|
||||
NodeStateUp = "UP"
|
||||
NodeStateDown = "DOWN"
|
||||
)
|
||||
|
||||
// Node represents a node in the cluster.
|
||||
type Node struct {
|
||||
Host string `json:"host"`
|
||||
InternalHost string `json:"internalHost"`
|
||||
|
||||
status *internal.NodeStatus `json:"status"`
|
||||
}
|
||||
|
||||
// SetStatus sets the NodeStatus.
|
||||
func (n *Node) SetStatus(s *internal.NodeStatus) {
|
||||
n.status = s
|
||||
}
|
||||
|
||||
// SetState sets the Node.status.state.
|
||||
func (n *Node) SetState(s string) {
|
||||
if n.status == nil {
|
||||
n.status = &internal.NodeStatus{}
|
||||
}
|
||||
n.status.State = s
|
||||
}
|
||||
|
||||
// Nodes represents a list of nodes.
|
||||
|
|
@ -124,21 +141,37 @@ func (c *Cluster) NodeSetHosts() []string {
|
|||
return a
|
||||
}
|
||||
|
||||
// Health returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value.
|
||||
func (c *Cluster) Health() map[string]string {
|
||||
// NodeStates returns a map of nodes in the cluster with each node's state (UP/DOWN) as the value.
|
||||
func (c *Cluster) NodeStates() map[string]string {
|
||||
h := make(map[string]string)
|
||||
for _, n := range c.Nodes {
|
||||
h[n.Host] = HealthStatusDown
|
||||
h[n.Host] = NodeStateDown
|
||||
}
|
||||
// we are assuming that NodeSetHosts is a subset of c.Nodes
|
||||
for _, m := range c.NodeSetHosts() {
|
||||
if _, ok := h[m]; ok {
|
||||
h[m] = HealthStatusUp
|
||||
h[m] = NodeStateUp
|
||||
}
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// State returns the internal ClusterState representation.
|
||||
func (c *Cluster) Status() *internal.ClusterStatus {
|
||||
return &internal.ClusterStatus{
|
||||
Nodes: encodeClusterStatus(c.Nodes),
|
||||
}
|
||||
}
|
||||
|
||||
// encodeClusterStatus converts a into its internal representation.
|
||||
func encodeClusterStatus(a []*Node) []*internal.NodeStatus {
|
||||
other := make([]*internal.NodeStatus, len(a))
|
||||
for i := range a {
|
||||
other[i] = a[i].status
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
||||
// NodeByHost returns a node reference by host.
|
||||
func (c *Cluster) NodeByHost(host string) *Node {
|
||||
for _, n := range c.Nodes {
|
||||
|
|
@ -150,25 +183,25 @@ func (c *Cluster) NodeByHost(host string) *Node {
|
|||
}
|
||||
|
||||
// Partition returns the partition that a slice belongs to.
|
||||
func (c *Cluster) Partition(db string, slice uint64) int {
|
||||
func (c *Cluster) Partition(index string, slice uint64) int {
|
||||
var buf [8]byte
|
||||
binary.BigEndian.PutUint64(buf[:], slice)
|
||||
|
||||
// Hash the bytes and mod by partition count.
|
||||
h := fnv.New64a()
|
||||
h.Write([]byte(db))
|
||||
h.Write([]byte(index))
|
||||
h.Write(buf[:])
|
||||
return int(h.Sum64() % uint64(c.PartitionN))
|
||||
}
|
||||
|
||||
// FragmentNodes returns a list of nodes that own a fragment.
|
||||
func (c *Cluster) FragmentNodes(db string, slice uint64) []*Node {
|
||||
return c.PartitionNodes(c.Partition(db, slice))
|
||||
func (c *Cluster) FragmentNodes(index string, slice uint64) []*Node {
|
||||
return c.PartitionNodes(c.Partition(index, slice))
|
||||
}
|
||||
|
||||
// OwnsFragment returns true if a host owns a fragment.
|
||||
func (c *Cluster) OwnsFragment(host string, db string, slice uint64) bool {
|
||||
return Nodes(c.FragmentNodes(db, slice)).ContainsHost(host)
|
||||
func (c *Cluster) OwnsFragment(host string, index string, slice uint64) bool {
|
||||
return Nodes(c.FragmentNodes(index, slice)).ContainsHost(host)
|
||||
}
|
||||
|
||||
// PartitionNodes returns a list of nodes that own a partition.
|
||||
|
|
@ -183,17 +216,32 @@ func (c *Cluster) PartitionNodes(partitionID int) []*Node {
|
|||
}
|
||||
|
||||
// Determine primary owner node.
|
||||
index := c.Hasher.Hash(uint64(partitionID), len(c.Nodes))
|
||||
node_index := c.Hasher.Hash(uint64(partitionID), len(c.Nodes))
|
||||
|
||||
// Collect nodes around the ring.
|
||||
nodes := make([]*Node, replicaN)
|
||||
for i := 0; i < replicaN; i++ {
|
||||
nodes[i] = c.Nodes[(index+i)%len(c.Nodes)]
|
||||
nodes[i] = c.Nodes[(node_index+i)%len(c.Nodes)]
|
||||
}
|
||||
|
||||
return nodes
|
||||
}
|
||||
|
||||
// OwnsSlices find the set of slices owned by the node per Index
|
||||
func (c *Cluster) OwnsSlices(index string, maxSlice uint64, host string) []uint64 {
|
||||
var slices []uint64
|
||||
for i := uint64(0); i <= maxSlice; i++ {
|
||||
p := c.Partition(index, i)
|
||||
// Determine primary owner node.
|
||||
node_index := c.Hasher.Hash(uint64(p), len(c.Nodes))
|
||||
if c.Nodes[node_index].Host == host {
|
||||
slices = append(slices, i)
|
||||
}
|
||||
|
||||
}
|
||||
return slices
|
||||
}
|
||||
|
||||
// Hasher represents an interface to hash integers into buckets.
|
||||
type Hasher interface {
|
||||
// Hashes the key into a number between [0,N).
|
||||
|
|
|
|||
|
|
@ -37,11 +37,11 @@ func TestCluster_Owners(t *testing.T) {
|
|||
|
||||
// Ensure the partitioner can assign a fragment to a partition.
|
||||
func TestCluster_Partition(t *testing.T) {
|
||||
if err := quick.Check(func(db string, slice uint64, partitionN int) bool {
|
||||
if err := quick.Check(func(index string, slice uint64, partitionN int) bool {
|
||||
c := pilosa.NewCluster()
|
||||
c.PartitionN = partitionN
|
||||
|
||||
partitionID := c.Partition(db, slice)
|
||||
partitionID := c.Partition(index, slice)
|
||||
if partitionID < 0 || partitionID >= partitionN {
|
||||
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
|
||||
}
|
||||
|
|
@ -89,7 +89,7 @@ func TestCluster_NodeSetHosts(t *testing.T) {
|
|||
}
|
||||
|
||||
// Ensure cluster can compare its Nodes and Members
|
||||
func TestCluster_Health(t *testing.T) {
|
||||
func TestCluster_NodeStates(t *testing.T) {
|
||||
c := pilosa.Cluster{
|
||||
Nodes: []*pilosa.Node{
|
||||
{Host: "serverA:1000"},
|
||||
|
|
@ -109,12 +109,12 @@ func TestCluster_Health(t *testing.T) {
|
|||
}
|
||||
|
||||
// Verify a DOWN node is reported, and extraneous nodes are ignored
|
||||
if a := c.Health(); !reflect.DeepEqual(a, map[string]string{
|
||||
"serverA:1000": pilosa.HealthStatusUp,
|
||||
"serverB:1000": pilosa.HealthStatusDown,
|
||||
"serverC:1000": pilosa.HealthStatusUp,
|
||||
if a := c.NodeStates(); !reflect.DeepEqual(a, map[string]string{
|
||||
"serverA:1000": pilosa.NodeStateUp,
|
||||
"serverB:1000": pilosa.NodeStateDown,
|
||||
"serverC:1000": pilosa.NodeStateUp,
|
||||
}) {
|
||||
t.Fatalf("unexpected health: %s", spew.Sdump(a))
|
||||
t.Fatalf("unexpected node state: %s", spew.Sdump(a))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ Backs up the view from across the cluster into a single file.
|
|||
}
|
||||
flags := backupCmd.Flags()
|
||||
flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
|
||||
flags.StringVarP(&Backuper.Database, "database", "d", "", "Pilosa database to backup into.")
|
||||
flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup into.")
|
||||
flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup into.")
|
||||
flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup into.")
|
||||
flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout")
|
||||
|
|
|
|||
|
|
@ -22,13 +22,13 @@ func TestBackupConfig(t *testing.T) {
|
|||
args: []string{"backup", "--output-file", "/somefile"},
|
||||
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
|
||||
cfgFileContent: `
|
||||
database = "mydb"
|
||||
index = "myindex"
|
||||
frame = "f1"
|
||||
`,
|
||||
validation: func() error {
|
||||
v := validator{}
|
||||
v.Check(cmd.Backuper.Host, "localhost:12345")
|
||||
v.Check(cmd.Backuper.Database, "mydb")
|
||||
v.Check(cmd.Backuper.Index, "myindex")
|
||||
v.Check(cmd.Backuper.Frame, "f1")
|
||||
v.Check(cmd.Backuper.Path, "/somefile")
|
||||
return v.Error()
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ func NewBenchCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
|
|||
Use: "bench",
|
||||
Short: "Benchmark operations.",
|
||||
Long: `
|
||||
Executes a benchmark for a given operation against the database.
|
||||
Executes a benchmark for a given operation against the index.
|
||||
`,
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if err := Bencher.Run(context.Background()); err != nil {
|
||||
|
|
@ -29,7 +29,7 @@ Executes a benchmark for a given operation against the database.
|
|||
}
|
||||
flags := benchCmd.Flags()
|
||||
flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
|
||||
flags.StringVarP(&Bencher.Database, "database", "d", "", "Pilosa database to benchmark.")
|
||||
flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.")
|
||||
flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.")
|
||||
flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]")
|
||||
flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.")
|
||||
|
|
|
|||
|
|
@ -22,13 +22,13 @@ func TestBenchConfig(t *testing.T) {
|
|||
args: []string{"bench", "--operation", "set-bit"},
|
||||
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
|
||||
cfgFileContent: `
|
||||
database = "mydb"
|
||||
index = "myindex"
|
||||
frame = "f1"
|
||||
`,
|
||||
validation: func() error {
|
||||
v := validator{}
|
||||
v.Check(cmd.Bencher.Host, "localhost:12345")
|
||||
v.Check(cmd.Bencher.Database, "mydb")
|
||||
v.Check(cmd.Bencher.Index, "myindex")
|
||||
v.Check(cmd.Bencher.Frame, "f1")
|
||||
v.Check(cmd.Bencher.Op, "set-bit")
|
||||
v.Check(cmd.Bencher.N, 0)
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@ the output is written to STDOUT.
|
|||
|
||||
The format of the CSV file is:
|
||||
|
||||
BITMAPID,PROFILEID
|
||||
ROWID,COLUMNID
|
||||
|
||||
The file does not contain any headers.
|
||||
`,
|
||||
|
|
@ -37,7 +37,7 @@ The file does not contain any headers.
|
|||
flags := exportCmd.Flags()
|
||||
|
||||
flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
|
||||
flags.StringVarP(&Exporter.Database, "database", "d", "", "Pilosa database to export into.")
|
||||
flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export into.")
|
||||
flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export into.")
|
||||
flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
|
||||
|
||||
|
|
|
|||
|
|
@ -22,13 +22,13 @@ func TestExportConfig(t *testing.T) {
|
|||
args: []string{"export", "--output-file", "/somefile"},
|
||||
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
|
||||
cfgFileContent: `
|
||||
database = "mydb"
|
||||
index = "myindex"
|
||||
frame = "f1"
|
||||
`,
|
||||
validation: func() error {
|
||||
v := validator{}
|
||||
v.Check(cmd.Exporter.Host, "localhost:12345")
|
||||
v.Check(cmd.Exporter.Database, "mydb")
|
||||
v.Check(cmd.Exporter.Index, "myindex")
|
||||
v.Check(cmd.Exporter.Frame, "f1")
|
||||
v.Check(cmd.Exporter.Path, "/somefile")
|
||||
return v.Error()
|
||||
|
|
|
|||
|
|
@ -16,12 +16,12 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
|
|||
importCmd := &cobra.Command{
|
||||
Use: "import",
|
||||
Short: "Bulk load data into pilosa.",
|
||||
Long: `Bulk imports one or more CSV files to a host's database and frame. The bits
|
||||
Long: `Bulk imports one or more CSV files to a host's index and frame. The bits
|
||||
of the CSV file are grouped by slice for the most efficient import.
|
||||
|
||||
The format of the CSV file is:
|
||||
|
||||
BITMAPID,PROFILEID,[TIME]
|
||||
ROWID,COLUMNID,[TIME]
|
||||
|
||||
The file should contain no headers. The TIME column is optional and can be
|
||||
omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
|
||||
|
|
@ -36,7 +36,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
|
|||
}
|
||||
flags := importCmd.Flags()
|
||||
flags.StringVarP(&Importer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
|
||||
flags.StringVarP(&Importer.Database, "database", "d", "", "Pilosa database to import into.")
|
||||
flags.StringVarP(&Importer.Index, "index", "i", "", "Pilosa index to import into.")
|
||||
flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.")
|
||||
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
|
||||
|
||||
|
|
|
|||
|
|
@ -22,13 +22,13 @@ func TestImportConfig(t *testing.T) {
|
|||
args: []string{"import"},
|
||||
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
|
||||
cfgFileContent: `
|
||||
database = "mydb"
|
||||
index = "myindex"
|
||||
frame = "f1"
|
||||
`,
|
||||
validation: func() error {
|
||||
v := validator{}
|
||||
v.Check(cmd.Importer.Host, "localhost:12345")
|
||||
v.Check(cmd.Importer.Database, "mydb")
|
||||
v.Check(cmd.Importer.Index, "myindex")
|
||||
v.Check(cmd.Importer.Frame, "f1")
|
||||
return v.Error()
|
||||
},
|
||||
|
|
|
|||
|
|
@ -30,10 +30,10 @@ Restores a view to the cluster from a backup file.
|
|||
}
|
||||
flags := restoreCmd.Flags()
|
||||
flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
|
||||
flags.StringVarP(&Restorer.Database, "database", "d", "", "Pilosa database to restore into.")
|
||||
flags.StringVarP(&Restorer.Index, "index", "i", "", "Pilosa index to restore into.")
|
||||
flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.")
|
||||
flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.")
|
||||
flags.StringVarP(&Restorer.Path, "input-file", "i", "", "File to restore from.")
|
||||
flags.StringVarP(&Restorer.Path, "input-file", "d", "", "File to restore data from.")
|
||||
|
||||
return restoreCmd
|
||||
}
|
||||
|
|
|
|||
|
|
@ -22,13 +22,13 @@ func TestRestoreConfig(t *testing.T) {
|
|||
args: []string{"restore", "--input-file", "/somefile"},
|
||||
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
|
||||
cfgFileContent: `
|
||||
database = "mydb"
|
||||
index = "myindex"
|
||||
frame = "f1"
|
||||
`,
|
||||
validation: func() error {
|
||||
v := validator{}
|
||||
v.Check(cmd.Restorer.Host, "localhost:12345")
|
||||
v.Check(cmd.Restorer.Database, "mydb")
|
||||
v.Check(cmd.Restorer.Index, "myindex")
|
||||
v.Check(cmd.Restorer.Frame, "f1")
|
||||
v.Check(cmd.Restorer.Path, "/somefile")
|
||||
return v.Error()
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ on the configured port.`,
|
|||
flags.StringVarP(&Server.Config.Cluster.Type, "cluster.type", "", "static", "Determine how the cluster handles membership and state sharing. Choose from [static, http, gossip]")
|
||||
flags.StringVarP(&Server.Config.Cluster.GossipSeed, "cluster.gossip-seed", "", "", "Host with which to seed the gossip membership.")
|
||||
flags.StringVarP(&Server.Config.Cluster.InternalPort, "cluster.internal-port", "", "", "Port to which pilosa should bind for internal state sharing.")
|
||||
flags.StringVarP(&Server.Config.Metric.Service, "metric.service", "", "noop", "Default URI on which pilosa should listen.")
|
||||
flags.StringVarP(&Server.Config.Metric.Service, "metric.service", "", "nop", "Default URI on which pilosa should listen.")
|
||||
flags.StringVarP(&Server.Config.Metric.Host, "metric.host", "", "", "Default URI to send metrics.")
|
||||
flags.DurationVarP((*time.Duration)(&Server.Config.Metric.PollingInterval), "metric.poll-interval", "", time.Minute*0, "Polling interval metrics.")
|
||||
return serveCmd
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ Sorts the import data at PATH into the optimal sort order for importing.
|
|||
|
||||
The format of the CSV file is:
|
||||
|
||||
BITMAPID,PROFILEID
|
||||
ROWID,COLUMNID
|
||||
|
||||
The file should contain no headers.
|
||||
`,
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ const (
|
|||
DefaultPort = "10101"
|
||||
DefaultClusterType = "static"
|
||||
DefaultInternalPort = "14000"
|
||||
DefaultMetrics = "noop"
|
||||
DefaultMetrics = "nop"
|
||||
)
|
||||
|
||||
// Config represents the configuration for the command.
|
||||
|
|
|
|||
|
|
@ -14,10 +14,10 @@ type BackupCommand struct {
|
|||
// Destination host and port.
|
||||
Host string
|
||||
|
||||
// Name of the database, frame, view to backup.
|
||||
Database string
|
||||
Frame string
|
||||
View string
|
||||
// Name of the index, frame, view to backup.
|
||||
Index string
|
||||
Frame string
|
||||
View string
|
||||
|
||||
// Output file to write to.
|
||||
Path string
|
||||
|
|
@ -54,7 +54,7 @@ func (cmd *BackupCommand) Run(ctx context.Context) error {
|
|||
defer f.Close()
|
||||
|
||||
// Begin streaming backup.
|
||||
if err := client.BackupTo(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil {
|
||||
if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
24
ctl/bench.go
24
ctl/bench.go
|
|
@ -11,14 +11,14 @@ import (
|
|||
"github.com/pilosa/pilosa"
|
||||
)
|
||||
|
||||
// BenchCommand represents a command for benchmarking database operations.
|
||||
// BenchCommand represents a command for benchmarking index operations.
|
||||
type BenchCommand struct {
|
||||
// Destination host and port.
|
||||
Host string
|
||||
|
||||
// Name of the database & frame to execute against.
|
||||
Database string
|
||||
Frame string
|
||||
// Name of the index & frame to execute against.
|
||||
Index string
|
||||
Frame string
|
||||
|
||||
// Type of operation and number to execute.
|
||||
Op string
|
||||
|
|
@ -57,25 +57,25 @@ func (cmd *BenchCommand) Run(ctx context.Context) error {
|
|||
func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) error {
|
||||
if cmd.N == 0 {
|
||||
return errors.New("operation count required")
|
||||
} else if cmd.Database == "" {
|
||||
return pilosa.ErrDatabaseRequired
|
||||
} else if cmd.Index == "" {
|
||||
return pilosa.ErrIndexRequired
|
||||
} else if cmd.Frame == "" {
|
||||
return pilosa.ErrFrameRequired
|
||||
}
|
||||
|
||||
const maxBitmapID = 1000
|
||||
const maxProfileID = 100000
|
||||
const maxRowID = 1000
|
||||
const maxColumnID = 100000
|
||||
|
||||
startTime := time.Now()
|
||||
|
||||
// Execute operation continuously.
|
||||
for i := 0; i < cmd.N; i++ {
|
||||
bitmapID := rand.Intn(maxBitmapID)
|
||||
profileID := rand.Intn(maxProfileID)
|
||||
rowID := rand.Intn(maxRowID)
|
||||
columnID := rand.Intn(maxColumnID)
|
||||
|
||||
q := fmt.Sprintf(`SetBit(id=%d, frame="%s", profileID=%d)`, bitmapID, cmd.Frame, profileID)
|
||||
q := fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID)
|
||||
|
||||
if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil {
|
||||
if _, err := client.ExecuteQuery(ctx, cmd.Index, q, true); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ bind = "localhost:10101"
|
|||
[anti-entropy]
|
||||
interval = "10m0s"
|
||||
|
||||
[metrics]
|
||||
[metric]
|
||||
service = "statsd"
|
||||
host = "127.0.0.1:8125"
|
||||
|
||||
|
|
|
|||
|
|
@ -14,9 +14,9 @@ type ExportCommand struct {
|
|||
// Remote host and port.
|
||||
Host string
|
||||
|
||||
// Name of the database & frame to export from.
|
||||
Database string
|
||||
Frame string
|
||||
// Name of the index & frame to export from.
|
||||
Index string
|
||||
Frame string
|
||||
|
||||
// Filename to export to.
|
||||
Path string
|
||||
|
|
@ -37,8 +37,8 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
|
|||
logger := log.New(cmd.Stderr, "", log.LstdFlags)
|
||||
|
||||
// Validate arguments.
|
||||
if cmd.Database == "" {
|
||||
return pilosa.ErrDatabaseRequired
|
||||
if cmd.Index == "" {
|
||||
return pilosa.ErrIndexRequired
|
||||
} else if cmd.Frame == "" {
|
||||
return pilosa.ErrFrameRequired
|
||||
}
|
||||
|
|
@ -63,15 +63,15 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
|
|||
}
|
||||
|
||||
// Determine slice count.
|
||||
maxSlices, err := client.MaxSliceByDatabase(ctx)
|
||||
maxSlices, err := client.MaxSliceByIndex(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Export each slice.
|
||||
for slice := uint64(0); slice <= maxSlices[cmd.Database]; slice++ {
|
||||
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
|
||||
logger.Printf("exporting slice: %d", slice)
|
||||
if err := client.ExportCSV(ctx, cmd.Database, cmd.Frame, slice, w); err != nil {
|
||||
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, slice, w); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@ type ImportCommand struct {
|
|||
// Destination host and port.
|
||||
Host string `json:"host"`
|
||||
|
||||
// Name of the database & frame to import into.
|
||||
Database string `json:"db"`
|
||||
Frame string `json:"frame"`
|
||||
// Name of the index & frame to import into.
|
||||
Index string `json:"index"`
|
||||
Frame string `json:"frame"`
|
||||
|
||||
// Filenames to import from.
|
||||
Paths []string `json:"paths"`
|
||||
|
|
@ -54,9 +54,9 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
|
|||
logger := log.New(cmd.Stderr, "", log.LstdFlags)
|
||||
|
||||
// Validate arguments.
|
||||
// Database and frame are validated early before the files are parsed.
|
||||
if cmd.Database == "" {
|
||||
return pilosa.ErrDatabaseRequired
|
||||
// Index and frame are validated early before the files are parsed.
|
||||
if cmd.Index == "" {
|
||||
return pilosa.ErrIndexRequired
|
||||
} else if cmd.Frame == "" {
|
||||
return pilosa.ErrFrameRequired
|
||||
} else if len(cmd.Paths) == 0 {
|
||||
|
|
@ -123,19 +123,19 @@ func (cmd *ImportCommand) importPath(ctx context.Context, path string) error {
|
|||
|
||||
var bit pilosa.Bit
|
||||
|
||||
// Parse bitmap id.
|
||||
bitmapID, err := strconv.ParseUint(record[0], 10, 64)
|
||||
// Parse row id.
|
||||
rowID, err := strconv.ParseUint(record[0], 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid bitmap id on row %d: %q", rnum, record[0])
|
||||
return fmt.Errorf("invalid row id on row %d: %q", rnum, record[0])
|
||||
}
|
||||
bit.BitmapID = bitmapID
|
||||
bit.RowID = rowID
|
||||
|
||||
// Parse bitmap id.
|
||||
profileID, err := strconv.ParseUint(record[1], 10, 64)
|
||||
// Parse column id.
|
||||
columnID, err := strconv.ParseUint(record[1], 10, 64)
|
||||
if err != nil {
|
||||
return fmt.Errorf("invalid profile id on row %d: %q", rnum, record[1])
|
||||
return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1])
|
||||
}
|
||||
bit.ProfileID = profileID
|
||||
bit.ColumnID = columnID
|
||||
|
||||
// Parse time, if exists.
|
||||
if len(record) > 2 && record[2] != "" {
|
||||
|
|
@ -176,7 +176,7 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
|
|||
// Parse path into bits.
|
||||
for slice, bits := range bitsBySlice {
|
||||
logger.Printf("importing slice: %d, n=%d", slice, len(bits))
|
||||
if err := cmd.Client.Import(ctx, cmd.Database, cmd.Frame, slice, bits); err != nil {
|
||||
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,10 +14,10 @@ type RestoreCommand struct {
|
|||
// Destination host and port.
|
||||
Host string
|
||||
|
||||
// Name of the database & frame to backup.
|
||||
Database string
|
||||
Frame string
|
||||
View string
|
||||
// Name of the index & frame to backup.
|
||||
Index string
|
||||
Frame string
|
||||
View string
|
||||
|
||||
// Import file to read from.
|
||||
Path string
|
||||
|
|
@ -54,7 +54,7 @@ func (cmd *RestoreCommand) Run(ctx context.Context) error {
|
|||
defer f.Close()
|
||||
|
||||
// Restore backup file to the cluster.
|
||||
if err := client.RestoreFrom(ctx, f, cmd.Database, cmd.Frame, cmd.View); err != nil {
|
||||
if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
|
|||
26
ctl/sort.go
26
ctl/sort.go
|
|
@ -45,7 +45,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error {
|
|||
r.FieldsPerRecord = -1
|
||||
a := make([]pilosa.Bit, 0, 1000000)
|
||||
for {
|
||||
bitmapID, profileID, timestamp, err := readCSVRow(r)
|
||||
rowID, columnID, timestamp, err := readCSVRow(r)
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err == errBlank {
|
||||
|
|
@ -53,7 +53,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error {
|
|||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp})
|
||||
a = append(a, pilosa.Bit{RowID: rowID, ColumnID: columnID, Timestamp: timestamp})
|
||||
}
|
||||
|
||||
// Sort bits by position.
|
||||
|
|
@ -65,10 +65,10 @@ func (cmd *SortCommand) Run(ctx context.Context) error {
|
|||
for _, bit := range a {
|
||||
// Write CSV to buffer.
|
||||
buf = buf[:0]
|
||||
buf = strconv.AppendUint(buf, bit.BitmapID, 10)
|
||||
buf = strconv.AppendUint(buf, bit.RowID, 10)
|
||||
|
||||
buf = append(buf, ',')
|
||||
buf = strconv.AppendUint(buf, bit.ProfileID, 10)
|
||||
buf = strconv.AppendUint(buf, bit.ColumnID, 10)
|
||||
|
||||
if bit.Timestamp != 0 {
|
||||
buf = append(buf, ',')
|
||||
|
|
@ -91,8 +91,8 @@ func (cmd *SortCommand) Run(ctx context.Context) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// readCSVRow reads a bitmap/profile pair from a CSV row.
|
||||
func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) {
|
||||
// readCSVRow reads a row/column pair from a CSV row.
|
||||
func readCSVRow(r *csv.Reader) (rowID, columnID uint64, timestamp int64, err error) {
|
||||
// Read CSV row.
|
||||
record, err := r.Read()
|
||||
if err != nil {
|
||||
|
|
@ -106,16 +106,16 @@ func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err
|
|||
return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record))
|
||||
}
|
||||
|
||||
// Parse bitmap id.
|
||||
bitmapID, err = strconv.ParseUint(record[0], 10, 64)
|
||||
// Parse row id.
|
||||
rowID, err = strconv.ParseUint(record[0], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0])
|
||||
return 0, 0, 0, fmt.Errorf("invalid row id: %q", record[0])
|
||||
}
|
||||
|
||||
// Parse bitmap id.
|
||||
profileID, err = strconv.ParseUint(record[1], 10, 64)
|
||||
// Parse column id.
|
||||
columnID, err = strconv.ParseUint(record[1], 10, 64)
|
||||
if err != nil {
|
||||
return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1])
|
||||
return 0, 0, 0, fmt.Errorf("invalid column id: %q", record[1])
|
||||
}
|
||||
|
||||
// Parse timestamp, if available.
|
||||
|
|
@ -127,7 +127,7 @@ func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err
|
|||
timestamp = t.UnixNano()
|
||||
}
|
||||
|
||||
return bitmapID, profileID, timestamp, nil
|
||||
return rowID, columnID, timestamp, nil
|
||||
}
|
||||
|
||||
// errBlank indicates a blank row in a CSV file.
|
||||
|
|
|
|||
|
|
@ -70,6 +70,14 @@ func (c *StatsClient) Count(name string, value int64) {
|
|||
}
|
||||
}
|
||||
|
||||
// Count tracks the number of times something occurs per second with custom tags.
|
||||
func (c *StatsClient) CountWithCustomTags(name string, value int64, t []string) {
|
||||
tags := append(c.tags, t...)
|
||||
if err := c.client.Count(name, value, tags, Rate); err != nil {
|
||||
c.logger().Printf("datadog.StatsClient.Count error: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Gauge sets the value of a metric.
|
||||
func (c *StatsClient) Gauge(name string, value float64) {
|
||||
if err := c.client.Gauge(name, value, c.tags, Rate); err != nil {
|
||||
|
|
|
|||
557
db.go
557
db.go
|
|
@ -1,557 +0,0 @@
|
|||
package pilosa
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/pilosa/pilosa/internal"
|
||||
)
|
||||
|
||||
// Default database settings.
|
||||
const (
|
||||
DefaultColumnLabel = "profileID"
|
||||
)
|
||||
|
||||
// DB represents a container for frames.
|
||||
type DB struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
name string
|
||||
|
||||
// Default time quantum for all frames in database.
|
||||
// This can be overridden by individual frames.
|
||||
timeQuantum TimeQuantum
|
||||
|
||||
// Label used for referring to columns in database.
|
||||
columnLabel string
|
||||
|
||||
// Frames by name.
|
||||
frames map[string]*Frame
|
||||
|
||||
// Max Slice on any node in the cluster, according to this node
|
||||
remoteMaxSlice uint64
|
||||
remoteMaxInverseSlice uint64
|
||||
|
||||
// Profile attribute storage and cache
|
||||
profileAttrStore *AttrStore
|
||||
|
||||
broadcaster Broadcaster
|
||||
stats StatsClient
|
||||
|
||||
LogOutput io.Writer
|
||||
}
|
||||
|
||||
// NewDB returns a new instance of DB.
|
||||
func NewDB(path, name string) (*DB, error) {
|
||||
err := ValidateName(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &DB{
|
||||
path: path,
|
||||
name: name,
|
||||
frames: make(map[string]*Frame),
|
||||
|
||||
remoteMaxSlice: 0,
|
||||
remoteMaxInverseSlice: 0,
|
||||
|
||||
profileAttrStore: NewAttrStore(filepath.Join(path, ".data")),
|
||||
|
||||
columnLabel: DefaultColumnLabel,
|
||||
|
||||
stats: NopStatsClient,
|
||||
LogOutput: ioutil.Discard,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Name returns name of the database.
|
||||
func (db *DB) Name() string { return db.name }
|
||||
|
||||
// Path returns the path the database was initialized with.
|
||||
func (db *DB) Path() string { return db.path }
|
||||
|
||||
// ProfileAttrStore returns the storage for profile attributes.
|
||||
func (db *DB) ProfileAttrStore() *AttrStore { return db.profileAttrStore }
|
||||
|
||||
// SetColumnLabel sets the column label. Persists to meta file on update.
|
||||
func (db *DB) SetColumnLabel(v string) error {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
// Ignore if no change occurred.
|
||||
if v == "" || db.columnLabel == v {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Make sure columnLabel is valid name
|
||||
err := ValidateName(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Persist meta data to disk on change.
|
||||
db.columnLabel = v
|
||||
if err := db.saveMeta(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ColumnLabel returns the column label.
|
||||
func (db *DB) ColumnLabel() string {
|
||||
db.mu.Lock()
|
||||
v := db.columnLabel
|
||||
db.mu.Unlock()
|
||||
return v
|
||||
}
|
||||
|
||||
// Open opens and initializes the database.
|
||||
func (db *DB) Open() error {
|
||||
// Ensure the path exists.
|
||||
if err := os.MkdirAll(db.path, 0777); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Read meta file.
|
||||
if err := db.loadMeta(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := db.openFrames(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := db.profileAttrStore.Open(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// openFrames opens and initializes the frames inside the database.
|
||||
func (db *DB) openFrames() error {
|
||||
f, err := os.Open(db.path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
fis, err := f.Readdir(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
if !fi.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
fr, err := db.newFrame(db.FramePath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
|
||||
if err != nil {
|
||||
return ErrName
|
||||
}
|
||||
if err := fr.Open(); err != nil {
|
||||
return fmt.Errorf("open frame: name=%s, err=%s", fr.Name(), err)
|
||||
}
|
||||
db.frames[fr.Name()] = fr
|
||||
|
||||
db.stats.Count("frameN", 1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadMeta reads meta data for the database, if any.
|
||||
func (db *DB) loadMeta() error {
|
||||
var pb internal.DBMeta
|
||||
|
||||
// Read data from meta file.
|
||||
buf, err := ioutil.ReadFile(filepath.Join(db.path, ".meta"))
|
||||
if os.IsNotExist(err) {
|
||||
db.timeQuantum = ""
|
||||
db.columnLabel = DefaultColumnLabel
|
||||
return nil
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else {
|
||||
if err := proto.Unmarshal(buf, &pb); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Copy metadata fields.
|
||||
db.timeQuantum = TimeQuantum(pb.TimeQuantum)
|
||||
db.columnLabel = pb.ColumnLabel
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// saveMeta writes meta data for the database.
|
||||
func (db *DB) saveMeta() error {
|
||||
// Marshal metadata.
|
||||
buf, err := proto.Marshal(&internal.DBMeta{
|
||||
TimeQuantum: string(db.timeQuantum),
|
||||
ColumnLabel: db.columnLabel,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Write to meta file.
|
||||
if err := ioutil.WriteFile(filepath.Join(db.path, ".meta"), buf, 0666); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes the database and its frames.
|
||||
func (db *DB) Close() error {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
// Close the attribute store.
|
||||
if db.profileAttrStore != nil {
|
||||
db.profileAttrStore.Close()
|
||||
}
|
||||
|
||||
// Close all frames.
|
||||
for _, f := range db.frames {
|
||||
f.Close()
|
||||
}
|
||||
db.frames = make(map[string]*Frame)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// MaxSlice returns the max slice in the database according to this node.
|
||||
func (db *DB) MaxSlice() uint64 {
|
||||
if db == nil {
|
||||
return 0
|
||||
}
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
max := db.remoteMaxSlice
|
||||
for _, f := range db.frames {
|
||||
if slice := f.MaxSlice(); slice > max {
|
||||
max = slice
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
func (db *DB) SetRemoteMaxSlice(newmax uint64) {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
db.remoteMaxSlice = newmax
|
||||
}
|
||||
|
||||
// MaxInverseSlice returns the max inverse slice in the database according to this node.
|
||||
func (db *DB) MaxInverseSlice() uint64 {
|
||||
if db == nil {
|
||||
return 0
|
||||
}
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
max := db.remoteMaxInverseSlice
|
||||
for _, f := range db.frames {
|
||||
if slice := f.MaxInverseSlice(); slice > max {
|
||||
max = slice
|
||||
}
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
func (db *DB) SetRemoteMaxInverseSlice(v uint64) {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
db.remoteMaxInverseSlice = v
|
||||
}
|
||||
|
||||
// TimeQuantum returns the default time quantum for the database.
|
||||
func (db *DB) TimeQuantum() TimeQuantum {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
return db.timeQuantum
|
||||
}
|
||||
|
||||
// SetTimeQuantum sets the default time quantum for the database.
|
||||
func (db *DB) SetTimeQuantum(q TimeQuantum) error {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
// Validate input.
|
||||
if !q.Valid() {
|
||||
return ErrInvalidTimeQuantum
|
||||
}
|
||||
|
||||
// Update value on database.
|
||||
db.timeQuantum = q
|
||||
|
||||
// Perist meta data to disk.
|
||||
if err := db.saveMeta(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FramePath returns the path to a frame in the database.
|
||||
func (db *DB) FramePath(name string) string { return filepath.Join(db.path, name) }
|
||||
|
||||
// Frame returns a frame in the database by name.
|
||||
func (db *DB) Frame(name string) *Frame {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
return db.frame(name)
|
||||
}
|
||||
|
||||
func (db *DB) frame(name string) *Frame { return db.frames[name] }
|
||||
|
||||
// Frames returns a list of all frames in the database.
|
||||
func (db *DB) Frames() []*Frame {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
a := make([]*Frame, 0, len(db.frames))
|
||||
for _, f := range db.frames {
|
||||
a = append(a, f)
|
||||
}
|
||||
sort.Sort(frameSlice(a))
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
// CreateFrame creates a frame.
|
||||
func (db *DB) CreateFrame(name string, opt FrameOptions) (*Frame, error) {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
// Ensure frame doesn't already exist.
|
||||
if db.frames[name] != nil {
|
||||
return nil, ErrFrameExists
|
||||
}
|
||||
return db.createFrame(name, opt)
|
||||
}
|
||||
|
||||
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
|
||||
func (db *DB) CreateFrameIfNotExists(name string, opt FrameOptions) (*Frame, error) {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
// Find frame in cache first.
|
||||
if f := db.frames[name]; f != nil {
|
||||
return f, nil
|
||||
}
|
||||
|
||||
return db.createFrame(name, opt)
|
||||
}
|
||||
|
||||
func (db *DB) createFrame(name string, opt FrameOptions) (*Frame, error) {
|
||||
if name == "" {
|
||||
return nil, errors.New("frame name required")
|
||||
} else if opt.CacheType != "" && !IsValidCacheType(opt.CacheType) {
|
||||
return nil, ErrInvalidCacheType
|
||||
}
|
||||
|
||||
// Initialize frame.
|
||||
f, err := db.newFrame(db.FramePath(name), name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Open frame.
|
||||
if err := f.Open(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Default the time quantum to what is set on the DB.
|
||||
if err := f.SetTimeQuantum(db.timeQuantum); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Set cache type.
|
||||
if opt.CacheType == "" {
|
||||
opt.CacheType = DefaultCacheType
|
||||
}
|
||||
f.cacheType = opt.CacheType
|
||||
|
||||
// Set options.
|
||||
if opt.RowLabel != "" {
|
||||
f.rowLabel = opt.RowLabel
|
||||
}
|
||||
if opt.CacheSize != 0 {
|
||||
f.cacheSize = opt.CacheSize
|
||||
}
|
||||
|
||||
f.inverseEnabled = opt.InverseEnabled
|
||||
if err := f.saveMeta(); err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Add to database's frame lookup.
|
||||
db.frames[name] = f
|
||||
|
||||
db.stats.Count("frameN", 1)
|
||||
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (db *DB) newFrame(path, name string) (*Frame, error) {
|
||||
f, err := NewFrame(path, db.name, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
f.LogOutput = db.LogOutput
|
||||
f.stats = db.stats.WithTags(fmt.Sprintf("frame:%s", name))
|
||||
f.broadcaster = db.broadcaster
|
||||
return f, nil
|
||||
}
|
||||
|
||||
// DeleteFrame removes a frame from the database.
|
||||
func (db *DB) DeleteFrame(name string) error {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
// Ignore if frame doesn't exist.
|
||||
f := db.frame(name)
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close frame.
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete frame directory.
|
||||
if err := os.RemoveAll(db.FramePath(name)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove reference.
|
||||
delete(db.frames, name)
|
||||
|
||||
db.stats.Count("frameN", -1)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type dbSlice []*DB
|
||||
|
||||
func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||
func (p dbSlice) Len() int { return len(p) }
|
||||
func (p dbSlice) Less(i, j int) bool { return p[i].Name() < p[j].Name() }
|
||||
|
||||
// DBInfo represents schema information for a database.
|
||||
type DBInfo struct {
|
||||
Name string `json:"name"`
|
||||
Frames []*FrameInfo `json:"frames"`
|
||||
}
|
||||
|
||||
type dbInfoSlice []*DBInfo
|
||||
|
||||
func (p dbInfoSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
|
||||
func (p dbInfoSlice) Len() int { return len(p) }
|
||||
func (p dbInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
|
||||
|
||||
// MergeSchemas combines databases and frames from a and b into one schema.
|
||||
func MergeSchemas(a, b []*DBInfo) []*DBInfo {
|
||||
// Generate a map from both schemas.
|
||||
m := make(map[string]map[string]map[string]struct{})
|
||||
for _, dbs := range [][]*DBInfo{a, b} {
|
||||
for _, db := range dbs {
|
||||
if m[db.Name] == nil {
|
||||
m[db.Name] = make(map[string]map[string]struct{})
|
||||
}
|
||||
for _, frame := range db.Frames {
|
||||
if m[db.Name][frame.Name] == nil {
|
||||
m[db.Name][frame.Name] = make(map[string]struct{})
|
||||
}
|
||||
for _, view := range frame.Views {
|
||||
m[db.Name][frame.Name][view.Name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Generate new schema from map.
|
||||
dbs := make([]*DBInfo, 0, len(m))
|
||||
for db, frames := range m {
|
||||
di := &DBInfo{Name: db}
|
||||
for frame, views := range frames {
|
||||
fi := &FrameInfo{Name: frame}
|
||||
for view := range views {
|
||||
fi.Views = append(fi.Views, &ViewInfo{Name: view})
|
||||
}
|
||||
sort.Sort(viewInfoSlice(fi.Views))
|
||||
di.Frames = append(di.Frames, fi)
|
||||
}
|
||||
sort.Sort(frameInfoSlice(di.Frames))
|
||||
dbs = append(dbs, di)
|
||||
}
|
||||
sort.Sort(dbInfoSlice(dbs))
|
||||
|
||||
return dbs
|
||||
}
|
||||
|
||||
// encodeDBs converts a into its internal representation.
|
||||
func encodeDBs(a []*DB) []*internal.DB {
|
||||
other := make([]*internal.DB, len(a))
|
||||
for i := range a {
|
||||
other[i] = encodeDB(a[i])
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
||||
// encodeDB converts d into its internal representation.
|
||||
func encodeDB(d *DB) *internal.DB {
|
||||
return &internal.DB{
|
||||
Name: d.name,
|
||||
Meta: &internal.DBMeta{
|
||||
ColumnLabel: d.columnLabel,
|
||||
TimeQuantum: string(d.timeQuantum),
|
||||
},
|
||||
MaxSlice: d.remoteMaxSlice,
|
||||
Frames: encodeFrames(d.Frames()),
|
||||
}
|
||||
}
|
||||
|
||||
// DBOptions represents options to set when initializing a db.
|
||||
type DBOptions struct {
|
||||
ColumnLabel string `json:"columnLabel,omitempty"`
|
||||
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
|
||||
}
|
||||
|
||||
// hasTime returns true if a contains a non-nil time.
|
||||
func hasTime(a []*time.Time) bool {
|
||||
for _, t := range a {
|
||||
if t != nil {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type importKey struct {
|
||||
View string
|
||||
Slice uint64
|
||||
}
|
||||
|
||||
type importData struct {
|
||||
BitmapIDs []uint64
|
||||
ProfileIDs []uint64
|
||||
}
|
||||
179
db_test.go
179
db_test.go
|
|
@ -1,179 +0,0 @@
|
|||
package pilosa_test
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
)
|
||||
|
||||
// Ensure database can open and retrieve a frame.
|
||||
func TestDB_CreateFrameIfNotExists(t *testing.T) {
|
||||
db := MustOpenDB()
|
||||
defer db.Close()
|
||||
|
||||
// Create frame.
|
||||
f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if f == nil {
|
||||
t.Fatal("expected frame")
|
||||
}
|
||||
|
||||
// Retrieve existing frame.
|
||||
other, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if f.Frame != other.Frame {
|
||||
t.Fatal("frame mismatch")
|
||||
}
|
||||
|
||||
if f.Frame != db.Frame("f") {
|
||||
t.Fatal("frame mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure database defaults the time quantum on new frames.
|
||||
func TestDB_CreateFrame_TimeQuantum(t *testing.T) {
|
||||
db := MustOpenDB()
|
||||
defer db.Close()
|
||||
|
||||
// Set database time quantum.
|
||||
if err := db.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Create frame.
|
||||
f, err := db.CreateFrame("f", pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") {
|
||||
t.Fatalf("unexpected frame time quantum: %s", q)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure database can delete a frame.
|
||||
func TestDB_DeleteFrame(t *testing.T) {
|
||||
db := MustOpenDB()
|
||||
defer db.Close()
|
||||
|
||||
// Create frame.
|
||||
if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Delete frame & verify it's gone.
|
||||
if err := db.DeleteFrame("f"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if db.Frame("f") != nil {
|
||||
t.Fatal("expected nil frame")
|
||||
}
|
||||
|
||||
// Delete again to make sure it doesn't error.
|
||||
if err := db.DeleteFrame("f"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure database can set the default time quantum.
|
||||
func TestDB_SetTimeQuantum(t *testing.T) {
|
||||
db := MustOpenDB()
|
||||
defer db.Close()
|
||||
|
||||
// Set & retrieve time quantum.
|
||||
if err := db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
|
||||
t.Fatalf("unexpected quantum: %s", q)
|
||||
}
|
||||
|
||||
// Reload database and verify that it is persisted.
|
||||
if err := db.Reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
|
||||
t.Fatalf("unexpected quantum (reopen): %s", q)
|
||||
}
|
||||
}
|
||||
|
||||
// DB represents a test wrapper for pilosa.DB.
|
||||
type DB struct {
|
||||
*pilosa.DB
|
||||
}
|
||||
|
||||
// NewDB returns a new instance of DB d.
|
||||
func NewDB() *DB {
|
||||
path, err := ioutil.TempDir("", "pilosa-db-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
db, err := pilosa.NewDB(path, "d")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &DB{DB: db}
|
||||
}
|
||||
|
||||
// MustOpenDB returns a new, opened database at a temporary path. Panic on error.
|
||||
func MustOpenDB() *DB {
|
||||
db := NewDB()
|
||||
if err := db.Open(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return db
|
||||
}
|
||||
|
||||
// Close closes the database and removes the underlying data.
|
||||
func (db *DB) Close() error {
|
||||
defer os.RemoveAll(db.Path())
|
||||
return db.DB.Close()
|
||||
}
|
||||
|
||||
// Reopen closes the database and reopens it.
|
||||
func (db *DB) Reopen() error {
|
||||
var err error
|
||||
if err := db.DB.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
path, name := db.Path(), db.Name()
|
||||
db.DB, err = pilosa.NewDB(path, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := db.Open(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// CreateFrame creates a frame with the given options.
|
||||
func (db *DB) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) {
|
||||
f, err := db.DB.CreateFrame(name, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Frame{Frame: f}, nil
|
||||
}
|
||||
|
||||
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
|
||||
func (db *DB) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) {
|
||||
f, err := db.DB.CreateFrameIfNotExists(name, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Frame{Frame: f}, nil
|
||||
}
|
||||
|
||||
// Ensure database can delete a frame.
|
||||
func TestDB_InvalidName(t *testing.T) {
|
||||
path, err := ioutil.TempDir("", "pilosa-db-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
db, err := pilosa.NewDB(path, "ABC")
|
||||
if db != nil {
|
||||
t.Fatalf("unexpected db name %s", db)
|
||||
}
|
||||
}
|
||||
290
executor.go
290
executor.go
|
|
@ -21,13 +21,13 @@ const (
|
|||
DefaultFrame = "general"
|
||||
|
||||
// MinThreshold is the lowest count to use in a Top-N operation when
|
||||
// looking for additional bitmap/count pairs.
|
||||
// looking for additional id/count pairs.
|
||||
MinThreshold = 1
|
||||
)
|
||||
|
||||
// Executor recursively executes calls in a PQL query across all slices.
|
||||
type Executor struct {
|
||||
Index *Index
|
||||
Holder *Holder
|
||||
|
||||
// Local hostname & cluster configuration.
|
||||
Host string
|
||||
|
|
@ -45,10 +45,10 @@ func NewExecutor() *Executor {
|
|||
}
|
||||
|
||||
// Execute executes a PQL query.
|
||||
func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) {
|
||||
// Verify that a database is set.
|
||||
if db == "" {
|
||||
return nil, ErrDatabaseRequired
|
||||
func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slices []uint64, opt *ExecOptions) ([]interface{}, error) {
|
||||
// Verify that an index is set.
|
||||
if index == "" {
|
||||
return nil, ErrIndexRequired
|
||||
}
|
||||
|
||||
// Default options.
|
||||
|
|
@ -60,7 +60,7 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices
|
|||
if len(slices) == 0 {
|
||||
if needsSlices(q.Calls) {
|
||||
// Round up the number of slices.
|
||||
maxSlice := e.Index.DB(db).MaxSlice()
|
||||
maxSlice := e.Holder.Index(index).MaxSlice()
|
||||
|
||||
// Generate a slices of all slices.
|
||||
slices = make([]uint64, maxSlice+1)
|
||||
|
|
@ -71,14 +71,14 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices
|
|||
}
|
||||
|
||||
// Optimize handling for bulk attribute insertion.
|
||||
if hasOnlySetBitmapAttrs(q.Calls) {
|
||||
return e.executeBulkSetBitmapAttrs(ctx, db, q.Calls, opt)
|
||||
if hasOnlySetRowAttrs(q.Calls) {
|
||||
return e.executeBulkSetRowAttrs(ctx, index, q.Calls, opt)
|
||||
}
|
||||
|
||||
// Execute each call serially.
|
||||
results := make([]interface{}, 0, len(q.Calls))
|
||||
for _, call := range q.Calls {
|
||||
v, err := e.executeCall(ctx, db, call, slices, opt)
|
||||
v, err := e.executeCall(ctx, index, call, slices, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -88,28 +88,31 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices
|
|||
}
|
||||
|
||||
// executeCall executes a call.
|
||||
func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
|
||||
func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
|
||||
|
||||
if err := e.validateCallArgs(c); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
indexTag := fmt.Sprintf("index:%s", index)
|
||||
// Special handling for mutation and top-n calls.
|
||||
switch c.Name {
|
||||
case "ClearBit":
|
||||
return e.executeClearBit(ctx, db, c, opt)
|
||||
return e.executeClearBit(ctx, index, c, opt)
|
||||
case "Count":
|
||||
return e.executeCount(ctx, db, c, slices, opt)
|
||||
e.Holder.Stats.CountWithCustomTags(c.Name, 1, []string{indexTag})
|
||||
return e.executeCount(ctx, index, c, slices, opt)
|
||||
case "SetBit":
|
||||
return e.executeSetBit(ctx, db, c, opt)
|
||||
case "SetBitmapAttrs":
|
||||
return nil, e.executeSetBitmapAttrs(ctx, db, c, opt)
|
||||
case "SetProfileAttrs":
|
||||
return nil, e.executeSetProfileAttrs(ctx, db, c, opt)
|
||||
return e.executeSetBit(ctx, index, c, opt)
|
||||
case "SetRowAttrs":
|
||||
return nil, e.executeSetRowAttrs(ctx, index, c, opt)
|
||||
case "SetColumnAttrs":
|
||||
return nil, e.executeSetColumnAttrs(ctx, index, c, opt)
|
||||
case "TopN":
|
||||
return e.executeTopN(ctx, db, c, slices, opt)
|
||||
e.Holder.Stats.CountWithCustomTags(c.Name, 1, []string{indexTag})
|
||||
return e.executeTopN(ctx, index, c, slices, opt)
|
||||
default:
|
||||
return e.executeBitmapCall(ctx, db, c, slices, opt)
|
||||
e.Holder.Stats.CountWithCustomTags(c.Name, 1, []string{indexTag})
|
||||
return e.executeBitmapCall(ctx, index, c, slices, opt)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -133,10 +136,10 @@ func (e *Executor) validateCallArgs(c *pql.Call) error {
|
|||
}
|
||||
|
||||
// executeBitmapCall executes a call that returns a bitmap.
|
||||
func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) {
|
||||
func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) {
|
||||
// Execute calls in bulk on each remote node and merge.
|
||||
mapFn := func(slice uint64) (interface{}, error) {
|
||||
return e.executeBitmapCallSlice(ctx, db, c, slice)
|
||||
return e.executeBitmapCallSlice(ctx, index, c, slice)
|
||||
}
|
||||
|
||||
// Merge returned results at coordinating node.
|
||||
|
|
@ -149,22 +152,22 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
|
|||
return other
|
||||
}
|
||||
|
||||
other, err := e.mapReduce(ctx, db, slices, c, opt, mapFn, reduceFn)
|
||||
other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Attach attributes for Bitmap() calls.
|
||||
// If the column label is used then return profile attributes.
|
||||
// If the column label is used then return column attributes.
|
||||
// If the row label is used then return bitmap attributes.
|
||||
bm, _ := other.(*Bitmap)
|
||||
if c.Name == "Bitmap" {
|
||||
|
||||
d := e.Index.DB(db)
|
||||
if d != nil {
|
||||
columnLabel := d.ColumnLabel()
|
||||
idx := e.Holder.Index(index)
|
||||
if idx != nil {
|
||||
columnLabel := idx.ColumnLabel()
|
||||
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
|
||||
attrs, err := d.ProfileAttrStore().Attrs(columnID)
|
||||
attrs, err := idx.ColumnAttrStore().Attrs(columnID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -173,13 +176,13 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
|
|||
return nil, err
|
||||
} else {
|
||||
frame, _ := c.Args["frame"].(string)
|
||||
if fr := d.Frame(frame); fr != nil {
|
||||
if fr := idx.Frame(frame); fr != nil {
|
||||
rowLabel := fr.RowLabel()
|
||||
rowID, _, err := c.UintArg(rowLabel)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
attrs, err := fr.BitmapAttrStore().Attrs(rowID)
|
||||
attrs, err := fr.RowAttrStore().Attrs(rowID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -193,18 +196,18 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
|
|||
}
|
||||
|
||||
// executeBitmapCallSlice executes a bitmap call for a single slice.
|
||||
func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
switch c.Name {
|
||||
case "Bitmap":
|
||||
return e.executeBitmapSlice(ctx, db, c, slice)
|
||||
return e.executeBitmapSlice(ctx, index, c, slice)
|
||||
case "Difference":
|
||||
return e.executeDifferenceSlice(ctx, db, c, slice)
|
||||
return e.executeDifferenceSlice(ctx, index, c, slice)
|
||||
case "Intersect":
|
||||
return e.executeIntersectSlice(ctx, db, c, slice)
|
||||
return e.executeIntersectSlice(ctx, index, c, slice)
|
||||
case "Range":
|
||||
return e.executeRangeSlice(ctx, db, c, slice)
|
||||
return e.executeRangeSlice(ctx, index, c, slice)
|
||||
case "Union":
|
||||
return e.executeUnionSlice(ctx, db, c, slice)
|
||||
return e.executeUnionSlice(ctx, index, c, slice)
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown call: %s", c.Name)
|
||||
}
|
||||
|
|
@ -213,8 +216,8 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql
|
|||
// executeTopN executes a TopN() call.
|
||||
// This first performs the TopN() to determine the top results and then
|
||||
// requeries to retrieve the full counts for each of the top results.
|
||||
func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
|
||||
bitmapIDs, _, err := c.UintSliceArg("ids")
|
||||
func (e *Executor) executeTopN(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
|
||||
rowIDs, _, err := c.UintSliceArg("ids")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("executeTopN: %v", err)
|
||||
}
|
||||
|
|
@ -224,14 +227,14 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic
|
|||
}
|
||||
|
||||
// Execute original query.
|
||||
pairs, err := e.executeTopNSlices(ctx, db, c, slices, opt)
|
||||
pairs, err := e.executeTopNSlices(ctx, index, c, slices, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// If this call is against specific ids, or we didn't get results,
|
||||
// or we are part of a larger distributed query then don't refetch.
|
||||
if len(pairs) == 0 || len(bitmapIDs) > 0 || opt.Remote {
|
||||
if len(pairs) == 0 || len(rowIDs) > 0 || opt.Remote {
|
||||
return pairs, nil
|
||||
}
|
||||
// Only the original caller should refetch the full counts.
|
||||
|
|
@ -241,7 +244,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic
|
|||
sort.Sort(uint64Slice(ids))
|
||||
other.Args["ids"] = ids
|
||||
|
||||
trimmedList, err := e.executeTopNSlices(ctx, db, other, slices, opt)
|
||||
trimmedList, err := e.executeTopNSlices(ctx, index, other, slices, opt)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -252,10 +255,10 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic
|
|||
return trimmedList, nil
|
||||
}
|
||||
|
||||
func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
|
||||
func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
|
||||
// Execute calls in bulk on each remote node and merge.
|
||||
mapFn := func(slice uint64) (interface{}, error) {
|
||||
return e.executeTopNSlice(ctx, db, c, slice)
|
||||
return e.executeTopNSlice(ctx, index, c, slice)
|
||||
}
|
||||
|
||||
// Merge returned results at coordinating node.
|
||||
|
|
@ -264,7 +267,7 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call
|
|||
return Pairs(other).Add(v.([]Pair))
|
||||
}
|
||||
|
||||
other, err := e.mapReduce(ctx, db, slices, c, opt, mapFn, reduceFn)
|
||||
other, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -277,14 +280,14 @@ func (e *Executor) executeTopNSlices(ctx context.Context, db string, c *pql.Call
|
|||
}
|
||||
|
||||
// executeTopNSlice executes a TopN call for a single slice.
|
||||
func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, slice uint64) ([]Pair, error) {
|
||||
func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
|
||||
frame, _ := c.Args["frame"].(string)
|
||||
n, _, err := c.UintArg("n")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("executeTopNSlice: %v", err)
|
||||
}
|
||||
field, _ := c.Args["field"].(string)
|
||||
bitmapIDs, _, err := c.UintSliceArg("ids")
|
||||
rowIDs, _, err := c.UintSliceArg("ids")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("executeTopNSlice: %v", err)
|
||||
}
|
||||
|
|
@ -301,7 +304,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
|
|||
// Retrieve bitmap used to intersect.
|
||||
var src *Bitmap
|
||||
if len(c.Children) == 1 {
|
||||
bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice)
|
||||
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -315,7 +318,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
|
|||
frame = DefaultFrame
|
||||
}
|
||||
|
||||
f := e.Index.Fragment(db, frame, ViewStandard, slice)
|
||||
f := e.Holder.Fragment(index, frame, ViewStandard, slice)
|
||||
if f == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
|
@ -330,7 +333,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
|
|||
return f.Top(TopOptions{
|
||||
N: int(n),
|
||||
Src: src,
|
||||
BitmapIDs: bitmapIDs,
|
||||
RowIDs: rowIDs,
|
||||
FilterField: field,
|
||||
FilterValues: filters,
|
||||
MinThreshold: minThreshold,
|
||||
|
|
@ -339,13 +342,13 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
|
|||
}
|
||||
|
||||
// executeDifferenceSlice executes a difference() call for a local slice.
|
||||
func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
func (e *Executor) executeDifferenceSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
var other *Bitmap
|
||||
if len(c.Children) == 0 {
|
||||
return nil, fmt.Errorf("empty Difference query is currently not supported")
|
||||
}
|
||||
for i, input := range c.Children {
|
||||
bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
|
||||
bm, err := e.executeBitmapCallSlice(ctx, index, input, slice)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -360,20 +363,20 @@ func (e *Executor) executeDifferenceSlice(ctx context.Context, db string, c *pql
|
|||
return other, nil
|
||||
}
|
||||
|
||||
func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
// Fetch column label from database.
|
||||
d := e.Index.DB(db)
|
||||
if d == nil {
|
||||
return nil, ErrDatabaseNotFound
|
||||
func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
// Fetch column label from index.
|
||||
idx := e.Holder.Index(index)
|
||||
if idx == nil {
|
||||
return nil, ErrIndexNotFound
|
||||
}
|
||||
columnLabel := d.ColumnLabel()
|
||||
columnLabel := idx.ColumnLabel()
|
||||
|
||||
// Fetch frame & row label based on argument.
|
||||
frame, _ := c.Args["frame"].(string)
|
||||
if frame == "" {
|
||||
frame = DefaultFrame
|
||||
}
|
||||
f := e.Index.Frame(db, frame)
|
||||
f := e.Holder.Frame(index, frame)
|
||||
if f == nil {
|
||||
return nil, ErrFrameNotFound
|
||||
}
|
||||
|
|
@ -400,21 +403,21 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal
|
|||
}
|
||||
}
|
||||
|
||||
frag := e.Index.Fragment(db, frame, view, slice)
|
||||
frag := e.Holder.Fragment(index, frame, view, slice)
|
||||
if frag == nil {
|
||||
return NewBitmap(), nil
|
||||
}
|
||||
return frag.Bitmap(id), nil
|
||||
return frag.Row(id), nil
|
||||
}
|
||||
|
||||
// executeIntersectSlice executes a intersect() call for a local slice.
|
||||
func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
func (e *Executor) executeIntersectSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
var other *Bitmap
|
||||
if len(c.Children) == 0 {
|
||||
return nil, fmt.Errorf("empty Intersect query is currently not supported")
|
||||
}
|
||||
for i, input := range c.Children {
|
||||
bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
|
||||
bm, err := e.executeBitmapCallSlice(ctx, index, input, slice)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -430,7 +433,7 @@ func (e *Executor) executeIntersectSlice(ctx context.Context, db string, c *pql.
|
|||
}
|
||||
|
||||
// executeRangeSlice executes a range() call for a local slice.
|
||||
func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
// Parse frame, use default if unset.
|
||||
frame, _ := c.Args["frame"].(string)
|
||||
if frame == "" {
|
||||
|
|
@ -438,7 +441,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
|
|||
}
|
||||
|
||||
// Retrieve base frame.
|
||||
f := e.Index.Frame(db, frame)
|
||||
f := e.Holder.Frame(index, frame)
|
||||
if f == nil {
|
||||
return nil, ErrFrameNotFound
|
||||
}
|
||||
|
|
@ -479,20 +482,21 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
|
|||
// Union bitmaps across all time-based subframes.
|
||||
bm := &Bitmap{}
|
||||
for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) {
|
||||
f := e.Index.Fragment(db, frame, view, slice)
|
||||
f := e.Holder.Fragment(index, frame, view, slice)
|
||||
if f == nil {
|
||||
continue
|
||||
}
|
||||
bm = bm.Union(f.Bitmap(rowID))
|
||||
bm = bm.Union(f.Row(rowID))
|
||||
}
|
||||
f.Stats.Count("range", 1)
|
||||
return bm, nil
|
||||
}
|
||||
|
||||
// executeUnionSlice executes a union() call for a local slice.
|
||||
func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
func (e *Executor) executeUnionSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (*Bitmap, error) {
|
||||
other := NewBitmap()
|
||||
for i, input := range c.Children {
|
||||
bm, err := e.executeBitmapCallSlice(ctx, db, input, slice)
|
||||
bm, err := e.executeBitmapCallSlice(ctx, index, input, slice)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -508,7 +512,7 @@ func (e *Executor) executeUnionSlice(ctx context.Context, db string, c *pql.Call
|
|||
}
|
||||
|
||||
// executeCount executes a count() call.
|
||||
func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) {
|
||||
func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (uint64, error) {
|
||||
if len(c.Children) == 0 {
|
||||
return 0, errors.New("Count() requires an input bitmap")
|
||||
} else if len(c.Children) > 1 {
|
||||
|
|
@ -517,7 +521,7 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, sli
|
|||
|
||||
// Execute calls in bulk on each remote node and merge.
|
||||
mapFn := func(slice uint64) (interface{}, error) {
|
||||
bm, err := e.executeBitmapCallSlice(ctx, db, c.Children[0], slice)
|
||||
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
|
@ -530,7 +534,7 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, sli
|
|||
return other + v.(uint64)
|
||||
}
|
||||
|
||||
result, err := e.mapReduce(ctx, db, slices, c, opt, mapFn, reduceFn)
|
||||
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
|
@ -540,7 +544,7 @@ func (e *Executor) executeCount(ctx context.Context, db string, c *pql.Call, sli
|
|||
}
|
||||
|
||||
// executeClearBit executes a ClearBit() call.
|
||||
func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) {
|
||||
func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
|
||||
view, _ := c.Args["view"].(string)
|
||||
frame, ok := c.Args["frame"].(string)
|
||||
if !ok {
|
||||
|
|
@ -548,17 +552,17 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
|
|||
}
|
||||
|
||||
// Retrieve frame.
|
||||
d := e.Index.DB(db)
|
||||
if d == nil {
|
||||
return false, ErrDatabaseNotFound
|
||||
idx := e.Holder.Index(index)
|
||||
if idx == nil {
|
||||
return false, ErrIndexNotFound
|
||||
}
|
||||
f := d.Frame(frame)
|
||||
f := idx.Frame(frame)
|
||||
if f == nil {
|
||||
return false, ErrFrameNotFound
|
||||
}
|
||||
|
||||
// Retrieve labels.
|
||||
columnLabel := d.ColumnLabel()
|
||||
columnLabel := idx.ColumnLabel()
|
||||
rowLabel := f.RowLabel()
|
||||
|
||||
// Read fields using labels.
|
||||
|
|
@ -579,19 +583,19 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
|
|||
// Clear bits for each view.
|
||||
switch view {
|
||||
case ViewStandard:
|
||||
return e.executeClearBitView(ctx, db, c, f, view, colID, rowID, opt)
|
||||
return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt)
|
||||
case ViewInverse:
|
||||
return e.executeClearBitView(ctx, db, c, f, view, rowID, colID, opt)
|
||||
return e.executeClearBitView(ctx, index, c, f, view, rowID, colID, opt)
|
||||
case "":
|
||||
var ret bool
|
||||
if changed, err := e.executeClearBitView(ctx, db, c, f, ViewStandard, colID, rowID, opt); err != nil {
|
||||
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt); err != nil {
|
||||
return ret, err
|
||||
} else if changed {
|
||||
ret = true
|
||||
}
|
||||
|
||||
if f.InverseEnabled() {
|
||||
if changed, err := e.executeClearBitView(ctx, db, c, f, ViewInverse, rowID, colID, opt); err != nil {
|
||||
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewInverse, rowID, colID, opt); err != nil {
|
||||
return ret, err
|
||||
} else if changed {
|
||||
ret = true
|
||||
|
|
@ -604,10 +608,10 @@ func (e *Executor) executeClearBit(ctx context.Context, db string, c *pql.Call,
|
|||
}
|
||||
|
||||
// executeClearBitView executes a ClearBit() call for a single view.
|
||||
func (e *Executor) executeClearBitView(ctx context.Context, db string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
|
||||
func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, opt *ExecOptions) (bool, error) {
|
||||
slice := colID / SliceWidth
|
||||
ret := false
|
||||
for _, node := range e.Cluster.FragmentNodes(db, slice) {
|
||||
for _, node := range e.Cluster.FragmentNodes(index, slice) {
|
||||
// Update locally if host matches.
|
||||
if node.Host == e.Host {
|
||||
val, err := f.ClearBit(view, rowID, colID, nil)
|
||||
|
|
@ -624,7 +628,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, db string, c *pql.Ca
|
|||
}
|
||||
|
||||
// Forward call to remote node otherwise.
|
||||
if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
|
||||
if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
|
||||
return false, err
|
||||
} else {
|
||||
ret = res[0].(bool)
|
||||
|
|
@ -634,7 +638,7 @@ func (e *Executor) executeClearBitView(ctx context.Context, db string, c *pql.Ca
|
|||
}
|
||||
|
||||
// executeSetBit executes a SetBit() call.
|
||||
func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) (bool, error) {
|
||||
func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
|
||||
view, _ := c.Args["view"].(string)
|
||||
frame, ok := c.Args["frame"].(string)
|
||||
if !ok {
|
||||
|
|
@ -642,17 +646,17 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
|
|||
}
|
||||
|
||||
// Retrieve frame.
|
||||
d := e.Index.DB(db)
|
||||
if d == nil {
|
||||
return false, ErrDatabaseNotFound
|
||||
idx := e.Holder.Index(index)
|
||||
if idx == nil {
|
||||
return false, ErrIndexNotFound
|
||||
}
|
||||
f := d.Frame(frame)
|
||||
f := idx.Frame(frame)
|
||||
if f == nil {
|
||||
return false, ErrFrameNotFound
|
||||
}
|
||||
|
||||
// Retrieve labels.
|
||||
columnLabel := d.ColumnLabel()
|
||||
columnLabel := idx.ColumnLabel()
|
||||
rowLabel := f.RowLabel()
|
||||
|
||||
// Read fields using labels.
|
||||
|
|
@ -683,19 +687,19 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
|
|||
// Set bits for each view.
|
||||
switch view {
|
||||
case ViewStandard:
|
||||
return e.executeSetBitView(ctx, db, c, f, view, colID, rowID, timestamp, opt)
|
||||
return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt)
|
||||
case ViewInverse:
|
||||
return e.executeSetBitView(ctx, db, c, f, view, rowID, colID, timestamp, opt)
|
||||
return e.executeSetBitView(ctx, index, c, f, view, rowID, colID, timestamp, opt)
|
||||
case "":
|
||||
var ret bool
|
||||
if changed, err := e.executeSetBitView(ctx, db, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil {
|
||||
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil {
|
||||
return ret, err
|
||||
} else if changed {
|
||||
ret = true
|
||||
}
|
||||
|
||||
if f.InverseEnabled() {
|
||||
if changed, err := e.executeSetBitView(ctx, db, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil {
|
||||
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil {
|
||||
return ret, err
|
||||
} else if changed {
|
||||
ret = true
|
||||
|
|
@ -708,11 +712,11 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.Call, op
|
|||
}
|
||||
|
||||
// executeSetBitView executes a SetBit() call for a specific view.
|
||||
func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) {
|
||||
func (e *Executor) executeSetBitView(ctx context.Context, index string, c *pql.Call, f *Frame, view string, colID, rowID uint64, timestamp *time.Time, opt *ExecOptions) (bool, error) {
|
||||
slice := colID / SliceWidth
|
||||
ret := false
|
||||
|
||||
for _, node := range e.Cluster.FragmentNodes(db, slice) {
|
||||
for _, node := range e.Cluster.FragmentNodes(index, slice) {
|
||||
// Update locally if host matches.
|
||||
if node.Host == e.Host {
|
||||
val, err := f.SetBit(view, rowID, colID, timestamp)
|
||||
|
|
@ -730,7 +734,7 @@ func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call
|
|||
}
|
||||
|
||||
// Forward call to remote node otherwise.
|
||||
if res, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
|
||||
if res, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt); err != nil {
|
||||
return false, err
|
||||
} else {
|
||||
ret = res[0].(bool)
|
||||
|
|
@ -739,15 +743,15 @@ func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call
|
|||
return ret, nil
|
||||
}
|
||||
|
||||
// executeSetBitmapAttrs executes a SetBitmapAttrs() call.
|
||||
func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
|
||||
// executeSetRowAttrs executes a SetRowAttrs() call.
|
||||
func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error {
|
||||
frameName, ok := c.Args["frame"].(string)
|
||||
if !ok {
|
||||
return errors.New("SetBitmapAttrs() frame required")
|
||||
return errors.New("SetRowAttrs() frame required")
|
||||
}
|
||||
|
||||
// Retrieve frame.
|
||||
frame := e.Index.Frame(db, frameName)
|
||||
frame := e.Holder.Frame(index, frameName)
|
||||
if frame == nil {
|
||||
return ErrFrameNotFound
|
||||
}
|
||||
|
|
@ -756,9 +760,9 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
|
|||
// Parse labels.
|
||||
rowID, ok, err := c.UintArg(rowLabel)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading SetBitmapAttrs() row: %v", err)
|
||||
return fmt.Errorf("reading SetRowAttrs() row: %v", err)
|
||||
} else if !ok {
|
||||
return fmt.Errorf("SetBitmapAttrs() row field '%v' required.", rowLabel)
|
||||
return fmt.Errorf("SetRowAttrs() row field '%v' required.", rowLabel)
|
||||
}
|
||||
|
||||
// Copy args and remove reserved fields.
|
||||
|
|
@ -767,9 +771,10 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
|
|||
delete(attrs, rowLabel)
|
||||
|
||||
// Set attributes.
|
||||
if err := frame.BitmapAttrStore().SetAttrs(rowID, attrs); err != nil {
|
||||
if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
|
||||
return err
|
||||
}
|
||||
frame.Stats.Count("SetBitmapAttrs", 1)
|
||||
|
||||
// Do not forward call if this is already being forwarded.
|
||||
if opt.Remote {
|
||||
|
|
@ -781,7 +786,7 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
|
|||
resp := make(chan error, len(nodes))
|
||||
for _, node := range nodes {
|
||||
go func(node *Node) {
|
||||
_, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
|
||||
_, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
|
||||
resp <- err
|
||||
}(node)
|
||||
}
|
||||
|
|
@ -796,18 +801,18 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
|
|||
return nil
|
||||
}
|
||||
|
||||
// executeBulkSetBitmapAttrs executes a set of SetBitmapAttrs() calls.
|
||||
func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) {
|
||||
// executeBulkSetRowAttrs executes a set of SetRowAttrs() calls.
|
||||
func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) {
|
||||
// Collect attributes by frame/id.
|
||||
m := make(map[string]map[uint64]map[string]interface{})
|
||||
for _, c := range calls {
|
||||
frame, ok := c.Args["frame"].(string)
|
||||
if !ok {
|
||||
return nil, errors.New("SetBitmapAttrs() frame required")
|
||||
return nil, errors.New("SetRowAttrs() frame required")
|
||||
}
|
||||
|
||||
// Retrieve frame.
|
||||
f := e.Index.Frame(db, frame)
|
||||
f := e.Holder.Frame(index, frame)
|
||||
if f == nil {
|
||||
return nil, ErrFrameNotFound
|
||||
}
|
||||
|
|
@ -815,9 +820,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
|
|||
|
||||
rowID, ok, err := c.UintArg(rowLabel)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("reading SetBitmapAttrs() row: %v", rowLabel)
|
||||
return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel)
|
||||
} else if !ok {
|
||||
return nil, fmt.Errorf("SetBitmapAttrs row field '%v' required.", rowLabel)
|
||||
return nil, fmt.Errorf("SetRowAttrs row field '%v' required.", rowLabel)
|
||||
}
|
||||
|
||||
// Copy args and remove reserved fields.
|
||||
|
|
@ -846,15 +851,16 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
|
|||
// Bulk insert attributes by frame.
|
||||
for name, frameMap := range m {
|
||||
// Retrieve frame.
|
||||
frame := e.Index.Frame(db, name)
|
||||
frame := e.Holder.Frame(index, name)
|
||||
if frame == nil {
|
||||
return nil, ErrFrameNotFound
|
||||
}
|
||||
|
||||
// Set attributes.
|
||||
if err := frame.BitmapAttrStore().SetBulkAttrs(frameMap); err != nil {
|
||||
if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
frame.Stats.Count("SetBitmapAttrs", 1)
|
||||
}
|
||||
|
||||
// Do not forward call if this is already being forwarded.
|
||||
|
|
@ -867,7 +873,7 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
|
|||
resp := make(chan error, len(nodes))
|
||||
for _, node := range nodes {
|
||||
go func(node *Node) {
|
||||
_, err := e.exec(ctx, node, db, &pql.Query{Calls: calls}, nil, opt)
|
||||
_, err := e.exec(ctx, node, index, &pql.Query{Calls: calls}, nil, opt)
|
||||
resp <- err
|
||||
}(node)
|
||||
}
|
||||
|
|
@ -883,22 +889,22 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
|
|||
return make([]interface{}, len(calls)), nil
|
||||
}
|
||||
|
||||
// executeSetProfileAttrs executes a SetProfileAttrs() call.
|
||||
func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
|
||||
// Retrieve database.
|
||||
d := e.Index.DB(db)
|
||||
if d == nil {
|
||||
return ErrDatabaseNotFound
|
||||
// executeSetColumnAttrs executes a SetColumnAttrs() call.
|
||||
func (e *Executor) executeSetColumnAttrs(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) error {
|
||||
// Retrieve index.
|
||||
idx := e.Holder.Index(index)
|
||||
if idx == nil {
|
||||
return ErrIndexNotFound
|
||||
}
|
||||
|
||||
var colName string
|
||||
id, okID, errID := c.UintArg("id")
|
||||
if errID != nil || !okID {
|
||||
// Retrieve columnLabel
|
||||
columnLabel := d.columnLabel
|
||||
columnLabel := idx.columnLabel
|
||||
col, okCol, errCol := c.UintArg(columnLabel)
|
||||
if errCol != nil || !okCol {
|
||||
return fmt.Errorf("reading SetProfileAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol)
|
||||
return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol)
|
||||
}
|
||||
id = col
|
||||
colName = columnLabel
|
||||
|
|
@ -911,10 +917,10 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql
|
|||
delete(attrs, colName)
|
||||
|
||||
// Set attributes.
|
||||
if err := d.ProfileAttrStore().SetAttrs(id, attrs); err != nil {
|
||||
if err := idx.ColumnAttrStore().SetAttrs(id, attrs); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
idx.Stats.Count("SetProfileAttrs", 1)
|
||||
// Do not forward call if this is already being forwarded.
|
||||
if opt.Remote {
|
||||
return nil
|
||||
|
|
@ -925,7 +931,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql
|
|||
resp := make(chan error, len(nodes))
|
||||
for _, node := range nodes {
|
||||
go func(node *Node) {
|
||||
_, err := e.exec(ctx, node, db, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
|
||||
_, err := e.exec(ctx, node, index, &pql.Query{Calls: []*pql.Call{c}}, nil, opt)
|
||||
resp <- err
|
||||
}(node)
|
||||
}
|
||||
|
|
@ -941,7 +947,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql
|
|||
}
|
||||
|
||||
// exec executes a PQL query remotely for a set of slices on a node.
|
||||
func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) {
|
||||
func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) {
|
||||
// Encode request object.
|
||||
pbreq := &internal.QueryRequest{
|
||||
Query: q.String(),
|
||||
|
|
@ -957,7 +963,7 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query
|
|||
req, err := http.NewRequest("POST", (&url.URL{
|
||||
Scheme: "http",
|
||||
Host: node.Host,
|
||||
Path: fmt.Sprintf("/db/%s/query", db),
|
||||
Path: fmt.Sprintf("/index/%s/query", index),
|
||||
}).String(), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -1012,8 +1018,8 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query
|
|||
v, err = pb.Results[i].Changed, nil
|
||||
case "ClearBit":
|
||||
v, err = pb.Results[i].Changed, nil
|
||||
case "SetBitmapAttrs":
|
||||
case "SetProfileAttrs":
|
||||
case "SetRowAttrs":
|
||||
case "SetColumnAttrs":
|
||||
default:
|
||||
v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil
|
||||
}
|
||||
|
|
@ -1028,12 +1034,12 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query
|
|||
|
||||
// slicesByNode returns a mapping of nodes to slices.
|
||||
// Returns errSliceUnavailable if a slice cannot be allocated to a node.
|
||||
func (e *Executor) slicesByNode(nodes []*Node, db string, slices []uint64) (map[*Node][]uint64, error) {
|
||||
func (e *Executor) slicesByNode(nodes []*Node, index string, slices []uint64) (map[*Node][]uint64, error) {
|
||||
m := make(map[*Node][]uint64)
|
||||
|
||||
loop:
|
||||
for _, slice := range slices {
|
||||
for _, node := range e.Cluster.FragmentNodes(db, slice) {
|
||||
for _, node := range e.Cluster.FragmentNodes(index, slice) {
|
||||
if Nodes(nodes).Contains(node) {
|
||||
m[node] = append(m[node], slice)
|
||||
continue loop
|
||||
|
|
@ -1048,7 +1054,7 @@ loop:
|
|||
//
|
||||
// If a mapping of slices to a node fails then the slices are resplit across
|
||||
// secondary nodes and retried. This continues to occur until all nodes are exhausted.
|
||||
func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
|
||||
func (e *Executor) mapReduce(ctx context.Context, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) (interface{}, error) {
|
||||
ch := make(chan mapResponse, 0)
|
||||
|
||||
// Wrap context with a cancel to kill goroutines on exit.
|
||||
|
|
@ -1067,7 +1073,7 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c
|
|||
}
|
||||
|
||||
// Start mapping across all primary owners.
|
||||
if err := e.mapper(ctx, ch, nodes, db, slices, c, opt, mapFn, reduceFn); err != nil {
|
||||
if err := e.mapper(ctx, ch, nodes, index, slices, c, opt, mapFn, reduceFn); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
|
|
@ -1086,7 +1092,7 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c
|
|||
nodes = Nodes(nodes).Filter(resp.node)
|
||||
|
||||
// Begin mapper against secondary nodes.
|
||||
if err := e.mapper(ctx, ch, nodes, db, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable {
|
||||
if err := e.mapper(ctx, ch, nodes, index, resp.slices, c, opt, mapFn, reduceFn); err == errSliceUnavailable {
|
||||
return nil, resp.err
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -1106,9 +1112,9 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c
|
|||
}
|
||||
}
|
||||
|
||||
func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, db string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error {
|
||||
func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Node, index string, slices []uint64, c *pql.Call, opt *ExecOptions, mapFn mapFunc, reduceFn reduceFunc) error {
|
||||
// Group slices together by nodes.
|
||||
m, err := e.slicesByNode(nodes, db, slices)
|
||||
m, err := e.slicesByNode(nodes, index, slices)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -1123,7 +1129,7 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
|
|||
resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
|
||||
} else if !opt.Remote {
|
||||
|
||||
results, err := e.exec(ctx, n, db, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
|
||||
results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
|
||||
if len(results) > 0 {
|
||||
resp.result = results[0]
|
||||
}
|
||||
|
|
@ -1208,14 +1214,14 @@ func decodeError(s string) error {
|
|||
return errors.New(s)
|
||||
}
|
||||
|
||||
// hasOnlySetBitmapAttrs returns true if calls only contains SetBitmapAttrs() calls.
|
||||
func hasOnlySetBitmapAttrs(calls []*pql.Call) bool {
|
||||
// hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls.
|
||||
func hasOnlySetRowAttrs(calls []*pql.Call) bool {
|
||||
if len(calls) == 0 {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, call := range calls {
|
||||
if call.Name != "SetBitmapAttrs" {
|
||||
if call.Name != "SetRowAttrs" {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
@ -1228,7 +1234,7 @@ func needsSlices(calls []*pql.Call) bool {
|
|||
}
|
||||
for _, call := range calls {
|
||||
switch call.Name {
|
||||
case "ClearBit", "SetBit", "SetBitmapAttrs", "SetProfileAttrs":
|
||||
case "ClearBit", "SetBit", "SetRowAttrs", "SetColumnAttrs":
|
||||
continue
|
||||
case "Count", "TopN":
|
||||
return true
|
||||
|
|
|
|||
468
executor_test.go
468
executor_test.go
|
|
@ -15,29 +15,29 @@ import (
|
|||
// Ensure a bitmap query can be executed.
|
||||
func TestExecutor_Execute_Bitmap(t *testing.T) {
|
||||
t.Run("Row", func(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
|
||||
f, err := db.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
|
||||
// Set bits.
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(``+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, 3)+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, SliceWidth+1)+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 20, SliceWidth+1),
|
||||
if _, err := e.Execute(context.Background(), "i", MustParse(``+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1),
|
||||
), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
|
||||
if err := f.RowAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
|
||||
t.Fatalf("unexpected bits: %+v", bits)
|
||||
|
|
@ -47,28 +47,28 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
|
|||
})
|
||||
|
||||
t.Run("Column", func(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
|
||||
if _, err := db.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
|
||||
// Set bits.
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(``+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, 3)+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, SliceWidth+1)+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 20, SliceWidth+1),
|
||||
if _, err := e.Execute(context.Background(), "i", MustParse(``+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+
|
||||
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1),
|
||||
), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.ProfileAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
|
||||
if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(fmt.Sprintf(`Bitmap(profileID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) {
|
||||
t.Fatalf("unexpected bits: %+v", bits)
|
||||
|
|
@ -80,16 +80,16 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
|
|||
|
||||
// Ensure a difference query can be executed.
|
||||
func TestExecutor_Execute_Difference(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
|
||||
t.Fatalf("unexpected bits: %+v", bits)
|
||||
|
|
@ -98,30 +98,30 @@ func TestExecutor_Execute_Difference(t *testing.T) {
|
|||
|
||||
// Ensure an empty difference query behaves properly.
|
||||
func TestExecutor_Execute_Empty_Difference(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Difference()`), nil, nil); err == nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Difference()`), nil, nil); err == nil {
|
||||
t.Fatalf("Empty Difference query should give error, but got %v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure an intersect query can be executed.
|
||||
func TestExecutor_Execute_Intersect(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
|
||||
t.Fatalf("unexpected bits: %+v", bits)
|
||||
|
|
@ -130,28 +130,28 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
|
|||
|
||||
// Ensure an empty intersect query behaves properly.
|
||||
func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Intersect()`), nil, nil); err == nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect()`), nil, nil); err == nil {
|
||||
t.Fatalf("Empty Intersect query should give error, but got %v", res)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure a union query can be executed.
|
||||
func TestExecutor_Execute_Union(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
|
||||
t.Fatalf("unexpected bits: %+v", bits)
|
||||
|
|
@ -160,12 +160,12 @@ func TestExecutor_Execute_Union(t *testing.T) {
|
|||
|
||||
// Ensure an empty union query behaves properly.
|
||||
func TestExecutor_Execute_Empty_Union(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
idx.MustCreateFragmentIfNotExists("d", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Union()`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Union()`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
|
||||
t.Fatalf("unexpected bits: %+v", bits)
|
||||
|
|
@ -174,14 +174,14 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
|
|||
|
||||
// Ensure a count query can be executed.
|
||||
func TestExecutor_Execute_Count(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res[0] != uint64(3) {
|
||||
t.Fatalf("unexpected n: %d", res[0])
|
||||
|
|
@ -190,16 +190,16 @@ func TestExecutor_Execute_Count(t *testing.T) {
|
|||
|
||||
// Ensure a set query can be executed.
|
||||
func TestExecutor_Execute_SetBit(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
|
||||
if n := f.Bitmap(11).Count(); n != 0 {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
|
||||
if n := f.Row(11).Count(); n != 0 {
|
||||
t.Fatalf("unexpected bitmap count: %d", n)
|
||||
}
|
||||
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil {
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
if !res[0].(bool) {
|
||||
|
|
@ -207,10 +207,10 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
if n := f.Bitmap(11).Count(); n != 1 {
|
||||
if n := f.Row(11).Count(); n != 1 {
|
||||
t.Fatalf("unexpected bitmap count: %d", n)
|
||||
}
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil {
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else {
|
||||
if res[0].(bool) {
|
||||
|
|
@ -219,37 +219,37 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure a SetBitmapAttrs() query can be executed.
|
||||
func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
// Ensure a SetRowAttrs() query can be executed.
|
||||
func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Create frames.
|
||||
db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
|
||||
if _, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := db.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil {
|
||||
} else if _, err := index.CreateFrameIfNotExists("xxx", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set two fields on f/10.
|
||||
// Also set fields on other bitmaps and frames to test isolation.
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil {
|
||||
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
|
||||
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
|
||||
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f := idx.Frame("d", "f")
|
||||
if m, err := f.BitmapAttrStore().Attrs(10); err != nil {
|
||||
f := hldr.Frame("i", "f")
|
||||
if m, err := f.RowAttrStore().Attrs(10); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
|
||||
t.Fatalf("unexpected bitmap attr: %#v", m)
|
||||
|
|
@ -258,23 +258,23 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
|
|||
|
||||
// Ensure a TopN() query can be executed.
|
||||
func TestExecutor_Execute_TopN(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Set bits for bitmaps 0, 10, & 20 across two slices.
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
// Set bits for rows 0, 10, & 20 across two slices.
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).SetBit(0, (5*SliceWidth)+100)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
|
||||
// Execute query.
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
|
||||
{ID: 0, Count: 5},
|
||||
|
|
@ -284,20 +284,20 @@ func TestExecutor_Execute_TopN(t *testing.T) {
|
|||
}
|
||||
}
|
||||
func TestExecutor_Execute_TopN_fill(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Set bits for bitmaps 0, 10, & 20 across two slices.
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
|
||||
// Set bits for rows 0, 10, & 20 across two slices.
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
|
||||
|
||||
// Execute query.
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
|
||||
{ID: 0, Count: 4},
|
||||
|
|
@ -308,30 +308,30 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
|
|||
|
||||
// Ensure
|
||||
func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(0, 2*SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(0, 3*SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).SetBit(0, 4*SliceWidth)
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(1, 1)
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(2, SliceWidth+1)
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).SetBit(3, 2*SliceWidth+1)
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
|
||||
|
||||
// Execute query.
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
|
||||
{ID: 0, Count: 5},
|
||||
|
|
@ -342,27 +342,27 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
|
|||
|
||||
// Ensure a TopN() query with a source bitmap can be executed.
|
||||
func TestExecutor_Execute_TopN_Src(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Set bits for bitmaps 0, 10, & 20 across two slices.
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
|
||||
// Set bits for rows 0, 10, & 20 across two slices.
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
|
||||
|
||||
// Create an intersecting bitmap.
|
||||
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
|
||||
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
|
||||
// Create an intersecting row.
|
||||
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
|
||||
|
||||
// Execute query.
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
|
||||
{ID: 20, Count: 3},
|
||||
|
|
@ -376,17 +376,17 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
|
|||
//Ensure TopN handles Attribute filters
|
||||
func TestExecutor_Execute_TopN_Attr(t *testing.T) {
|
||||
//
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
|
||||
if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
|
||||
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
|
||||
{ID: 10, Count: 1},
|
||||
|
|
@ -399,17 +399,17 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
|
|||
//Ensure TopN handles Attribute filters with source bitmap
|
||||
func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
|
||||
//
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
|
||||
|
||||
if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
|
||||
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "d", MustParse(`TopN(Bitmap(id=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(id=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
|
||||
{ID: 10, Count: 1},
|
||||
|
|
@ -421,14 +421,14 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
|
|||
|
||||
// Ensure a range query can be executed.
|
||||
func TestExecutor_Execute_Range(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Create database.
|
||||
db := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
|
||||
// Create index.
|
||||
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
|
||||
// Create frame.
|
||||
f, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
|
||||
f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
|
||||
|
|
@ -445,10 +445,10 @@ func TestExecutor_Execute_Range(t *testing.T) {
|
|||
|
||||
f.MustSetBit(pilosa.ViewStandard, 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early
|
||||
f.MustSetBit(pilosa.ViewStandard, 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late
|
||||
f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap
|
||||
f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different row
|
||||
|
||||
e := NewExecutor(idx.Index, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) {
|
||||
t.Fatalf("unexpected bits: %+v", bits)
|
||||
|
|
@ -465,14 +465,12 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
|
|||
c.Nodes[1].Host = s.Host()
|
||||
|
||||
// Mock secondary server's executor to verify arguments and return a bitmap.
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if db != `d` {
|
||||
t.Fatalf("unexpected db: %s", db)
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if index != "i" {
|
||||
t.Fatalf("unexpected index: %s", index)
|
||||
} else if query.String() != `Bitmap(frame="f", id=10)` {
|
||||
t.Fatalf("unexpected query: %s", query.String())
|
||||
// NOTE: while the following is technically incorrect (it should be {0, 2}) because the calling node doesn't know about slice 2 yet,
|
||||
// we are ok with this and assuming that the calling node will become aware of slice 2 via inter-node messaging
|
||||
} else if !reflect.DeepEqual(slices, []uint64{0}) {
|
||||
} else if !reflect.DeepEqual(slices, []uint64{1}) {
|
||||
t.Fatalf("unexpected slices: %+v", slices)
|
||||
}
|
||||
|
||||
|
|
@ -487,15 +485,15 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
|
|||
|
||||
// Create local executor data.
|
||||
// The local node owns slice 1.
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
s.Handler.Index = idx.Index
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
s.Handler.Holder = hldr.Holder
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
|
||||
|
||||
e := NewExecutor(idx.Index, c)
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, c)
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, (1 * SliceWidth) + 1, 2*SliceWidth + 4}) {
|
||||
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) {
|
||||
t.Fatalf("unexpected bits: %+v", bits)
|
||||
}
|
||||
}
|
||||
|
|
@ -510,19 +508,19 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
|
|||
c.Nodes[1].Host = s.Host()
|
||||
|
||||
// Mock secondary server's executor to return a count.
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
return []interface{}{uint64(10)}, nil
|
||||
}
|
||||
|
||||
// Create local executor data. The local node owns slice 1.
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
s.Handler.Index = idx.Index
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+2)
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
s.Handler.Holder = hldr.Holder
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2)
|
||||
|
||||
e := NewExecutor(idx.Index, c)
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, c)
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res[0] != uint64(12) {
|
||||
t.Fatalf("unexpected n: %d", res[0])
|
||||
|
|
@ -541,10 +539,10 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
|
|||
|
||||
// Mock secondary server's executor to verify arguments.
|
||||
var remoteCalled bool
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if db != `d` {
|
||||
t.Fatalf("unexpected db: %s", db)
|
||||
} else if query.String() != `SetBit(frame="f", id=10, profileID=2)` {
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if index != `i` {
|
||||
t.Fatalf("unexpected index: %s", index)
|
||||
} else if query.String() != `SetBit(columnID=2, frame="f", id=10)` {
|
||||
t.Fatalf("unexpected query: %s", query.String())
|
||||
}
|
||||
remoteCalled = true
|
||||
|
|
@ -552,22 +550,22 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
|
|||
}
|
||||
|
||||
// Create local executor data.
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
s.Handler.Index = idx.Index
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
s.Handler.Holder = hldr.Holder
|
||||
|
||||
// Create frame.
|
||||
if _, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
|
||||
if _, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
e := NewExecutor(idx.Index, c)
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, c)
|
||||
if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify that one bit is set on both node's index.
|
||||
if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Bitmap(10).Count(); n != 1 {
|
||||
// Verify that one bit is set on both node's holder.
|
||||
if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 {
|
||||
t.Fatalf("unexpected local count: %d", n)
|
||||
}
|
||||
if !remoteCalled {
|
||||
|
|
@ -587,10 +585,10 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
|
|||
|
||||
// Mock secondary server's executor to verify arguments.
|
||||
var remoteCalled bool
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if db != `d` {
|
||||
t.Fatalf("unexpected db: %s", db)
|
||||
} else if query.String() != `SetBit(frame="f", id=10, profileID=2, timestamp="2016-12-11T10:09")` {
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if index != `i` {
|
||||
t.Fatalf("unexpected index: %s", index)
|
||||
} else if query.String() != `SetBit(columnID=2, frame="f", id=10, timestamp="2016-12-11T10:09")` {
|
||||
t.Fatalf("unexpected query: %s", query.String())
|
||||
}
|
||||
remoteCalled = true
|
||||
|
|
@ -598,24 +596,24 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
|
|||
}
|
||||
|
||||
// Create local executor data.
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
s.Handler.Index = idx.Index
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
s.Handler.Holder = hldr.Holder
|
||||
|
||||
// Create frame.
|
||||
if f, err := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
|
||||
if f, err := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}).CreateFrame("f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := f.SetTimeQuantum("Y"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
e := NewExecutor(idx.Index, c)
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, c)
|
||||
if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify that one bit is set on both node's index.
|
||||
if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Bitmap(10).Count(); n != 1 {
|
||||
// Verify that one bit is set on both node's holder.
|
||||
if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 {
|
||||
t.Fatalf("unexpected local count: %d", n)
|
||||
}
|
||||
if !remoteCalled {
|
||||
|
|
@ -634,10 +632,10 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
|
|||
|
||||
// Mock secondary server's executor to verify arguments and return a bitmap.
|
||||
var remoteExecN int
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if db != `d` {
|
||||
t.Fatalf("unexpected db: %s", db)
|
||||
} else if !reflect.DeepEqual(slices, []uint64{0, 2}) {
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if index != "i" {
|
||||
t.Fatalf("unexpected index: %s", index)
|
||||
} else if !reflect.DeepEqual(slices, []uint64{1, 3}) {
|
||||
t.Fatalf("unexpected slices: %+v", slices)
|
||||
}
|
||||
|
||||
|
|
@ -665,15 +663,15 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
|
|||
}}, nil
|
||||
}
|
||||
|
||||
// Create local executor data on slice 1 & 3.
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
s.Handler.Index = idx.Index
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
|
||||
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+2)
|
||||
// Create local executor data on slice 2 & 4.
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
s.Handler.Holder = hldr.Holder
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
|
||||
|
||||
e := NewExecutor(idx.Index, c)
|
||||
if res, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
|
||||
e := NewExecutor(hldr.Holder, c)
|
||||
if res, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{
|
||||
{ID: 0, Count: 5},
|
||||
|
|
@ -691,9 +689,9 @@ type Executor struct {
|
|||
|
||||
// NewExecutor returns a new instance of Executor.
|
||||
// The executor always matches the hostname of the first cluster node.
|
||||
func NewExecutor(index *pilosa.Index, cluster *pilosa.Cluster) *Executor {
|
||||
func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
|
||||
e := &Executor{Executor: pilosa.NewExecutor()}
|
||||
e.Index = index
|
||||
e.Holder = holder
|
||||
e.Cluster = cluster
|
||||
e.Host = cluster.Nodes[0].Host
|
||||
return e
|
||||
|
|
|
|||
326
fragment.go
326
fragment.go
|
|
@ -29,7 +29,7 @@ import (
|
|||
)
|
||||
|
||||
const (
|
||||
// SliceWidth is the number of profile IDs in a slice.
|
||||
// SliceWidth is the number of column IDs in a slice.
|
||||
SliceWidth = 1048576
|
||||
|
||||
// SnapshotExt is the file extension used for an in-process snapshot.
|
||||
|
|
@ -41,7 +41,7 @@ const (
|
|||
// CacheExt is the file extension for persisted cache ids.
|
||||
CacheExt = ".cache"
|
||||
|
||||
// HashBlockSize is the number of bitmaps in a merkle hash block.
|
||||
// HashBlockSize is the number of rows in a merkle hash block.
|
||||
HashBlockSize = 100
|
||||
)
|
||||
|
||||
|
|
@ -50,12 +50,12 @@ const (
|
|||
DefaultFragmentMaxOpN = 2000
|
||||
)
|
||||
|
||||
// Fragment represents the intersection of a frame and slice in a database.
|
||||
// Fragment represents the intersection of a frame and slice in an index.
|
||||
type Fragment struct {
|
||||
mu sync.Mutex
|
||||
|
||||
// Composite identifiers
|
||||
db string
|
||||
index string
|
||||
frame string
|
||||
view string
|
||||
slice uint64
|
||||
|
|
@ -67,13 +67,13 @@ type Fragment struct {
|
|||
storageData []byte
|
||||
opN int // number of ops since snapshot
|
||||
|
||||
// Cache for bitmap counts.
|
||||
// Cache for row counts.
|
||||
cacheType string // passed in by frame
|
||||
cache Cache
|
||||
cacheSize uint32
|
||||
|
||||
// Cache containing full bitmaps (not just counts).
|
||||
bitmapCache BitmapCache
|
||||
// Cache containing full rows (not just counts).
|
||||
rowCache BitmapCache
|
||||
|
||||
// Cached checksums for each block.
|
||||
checksums map[int][]byte
|
||||
|
|
@ -86,18 +86,18 @@ type Fragment struct {
|
|||
// Writer used for out-of-band log entries.
|
||||
LogOutput io.Writer
|
||||
|
||||
// Bitmap attribute storage.
|
||||
// Row attribute storage.
|
||||
// This is set by the parent frame unless overridden for testing.
|
||||
BitmapAttrStore *AttrStore
|
||||
RowAttrStore *AttrStore
|
||||
|
||||
stats StatsClient
|
||||
}
|
||||
|
||||
// NewFragment returns a new instance of Fragment.
|
||||
func NewFragment(path, db, frame, view string, slice uint64) *Fragment {
|
||||
func NewFragment(path, index, frame, view string, slice uint64) *Fragment {
|
||||
return &Fragment{
|
||||
path: path,
|
||||
db: db,
|
||||
index: index,
|
||||
frame: frame,
|
||||
view: view,
|
||||
slice: slice,
|
||||
|
|
@ -117,8 +117,8 @@ func (f *Fragment) Path() string { return f.path }
|
|||
// CachePath returns the path to the fragment's cache data.
|
||||
func (f *Fragment) CachePath() string { return f.path + CacheExt }
|
||||
|
||||
// DB returns the database the fragment was initialized with.
|
||||
func (f *Fragment) DB() string { return f.db }
|
||||
// Index returns the index that the fragment was initialized with.
|
||||
func (f *Fragment) Index() string { return f.index }
|
||||
|
||||
// Frame returns the frame the fragment was initialized with.
|
||||
func (f *Fragment) Frame() string { return f.frame }
|
||||
|
|
@ -144,7 +144,7 @@ func (f *Fragment) Open() error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Fill cache with bitmaps persisted to disk.
|
||||
// Fill cache with rows persisted to disk.
|
||||
if err := f.openCache(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -213,13 +213,13 @@ func (f *Fragment) openStorage() error {
|
|||
|
||||
// Attach the file to the bitmap to act as a write-ahead log.
|
||||
f.storage.OpWriter = f.file
|
||||
f.bitmapCache = &SimpleCache{make(map[uint64]*Bitmap)}
|
||||
f.rowCache = &SimpleCache{make(map[uint64]*Bitmap)}
|
||||
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// openCache initializes the cache from bitmap ids persisted to disk.
|
||||
// openCache initializes the cache from row ids persisted to disk.
|
||||
func (f *Fragment) openCache() error {
|
||||
// Determine cache type from frame name.
|
||||
switch f.cacheType {
|
||||
|
|
@ -247,12 +247,12 @@ func (f *Fragment) openCache() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Read in all bitmaps by ID.
|
||||
// Read in all rows by ID.
|
||||
// This will cause them to be added to the cache.
|
||||
for _, bitmapID := range pb.BitmapIDs {
|
||||
//n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth)
|
||||
n := f.bitmap(bitmapID, true, true).Count()
|
||||
f.cache.BulkAdd(bitmapID, n)
|
||||
for _, id := range pb.IDs {
|
||||
//n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth)
|
||||
n := f.row(id, true, true).Count()
|
||||
f.cache.BulkAdd(id, n)
|
||||
}
|
||||
f.cache.Invalidate()
|
||||
|
||||
|
|
@ -314,17 +314,16 @@ func (f *Fragment) closeStorage() error {
|
|||
// logger returns a logger instance for the fragment.nt.
|
||||
func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.LstdFlags) }
|
||||
|
||||
// Bitmap returns a bitmap by ID.
|
||||
func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap {
|
||||
// Row returns a row by ID.
|
||||
func (f *Fragment) Row(rowID uint64) *Bitmap {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.bitmap(bitmapID, true, true)
|
||||
return f.row(rowID, true, true)
|
||||
}
|
||||
|
||||
func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCache bool) *Bitmap {
|
||||
|
||||
if checkBitmapCache {
|
||||
r, ok := f.bitmapCache.Fetch(bitmapID)
|
||||
func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Bitmap {
|
||||
if checkRowCache {
|
||||
r, ok := f.rowCache.Fetch(rowID)
|
||||
if ok && r != nil {
|
||||
return r
|
||||
}
|
||||
|
|
@ -332,11 +331,11 @@ func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCa
|
|||
|
||||
// Only use a subset of the containers.
|
||||
// NOTE: The start & end ranges must be divisible by
|
||||
data := f.storage.OffsetRange(f.slice*SliceWidth, bitmapID*SliceWidth, (bitmapID+1)*SliceWidth)
|
||||
data := f.storage.OffsetRange(f.slice*SliceWidth, rowID*SliceWidth, (rowID+1)*SliceWidth)
|
||||
|
||||
// Reference bitmap subrange in storage.
|
||||
// We Clone() data because otherwise bm will contains pointers to containers in storage.
|
||||
// This causes unexpected results when we cache the bitmap and try to use it later.
|
||||
// This causes unexpected results when we cache the row and try to use it later.
|
||||
bm := &Bitmap{
|
||||
segments: []BitmapSegment{{
|
||||
data: *data.Clone(),
|
||||
|
|
@ -346,25 +345,25 @@ func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCa
|
|||
}
|
||||
bm.InvalidateCount()
|
||||
|
||||
if updateBitmapCache {
|
||||
f.bitmapCache.Add(bitmapID, bm)
|
||||
if updateRowCache {
|
||||
f.rowCache.Add(rowID, bm)
|
||||
}
|
||||
|
||||
return bm
|
||||
}
|
||||
|
||||
// SetBit sets a bit for a given profile & bitmap within the fragment.
|
||||
// SetBit sets a bit for a given column & row within the fragment.
|
||||
// This updates both the on-disk storage and the in-cache bitmap.
|
||||
func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) {
|
||||
func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.setBit(bitmapID, profileID)
|
||||
return f.setBit(rowID, columnID)
|
||||
}
|
||||
|
||||
func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error) {
|
||||
func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
|
||||
changed = false
|
||||
// Determine the position of the bit in the storage.
|
||||
pos, err := f.pos(bitmapID, profileID)
|
||||
pos, err := f.pos(rowID, columnID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
|
@ -380,37 +379,37 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error)
|
|||
}
|
||||
|
||||
// Invalidate block checksum.
|
||||
delete(f.checksums, int(bitmapID/HashBlockSize))
|
||||
delete(f.checksums, int(rowID/HashBlockSize))
|
||||
|
||||
// Increment number of operations until snapshot is required.
|
||||
if err := f.incrementOpN(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Get the bitmap from bitmapCache or fragment.storage.
|
||||
bm := f.bitmap(bitmapID, true, true)
|
||||
bm.SetBit(profileID)
|
||||
// Get the row from row cache or fragment.storage.
|
||||
bm := f.row(rowID, true, true)
|
||||
bm.SetBit(columnID)
|
||||
|
||||
// Update the cache.
|
||||
f.cache.Add(bitmapID, bm.Count())
|
||||
f.cache.Add(rowID, bm.Count())
|
||||
|
||||
f.stats.Count("setN", 1)
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// ClearBit clears a bit for a given profile & bitmap within the fragment.
|
||||
// ClearBit clears a bit for a given column & row within the fragment.
|
||||
// This updates both the on-disk storage and the in-cache bitmap.
|
||||
func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) {
|
||||
func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return f.clearBit(bitmapID, profileID)
|
||||
return f.clearBit(rowID, columnID)
|
||||
}
|
||||
|
||||
func (f *Fragment) clearBit(bitmapID, profileID uint64) (changed bool, err error) {
|
||||
func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
|
||||
changed = false
|
||||
// Determine the position of the bit in the storage.
|
||||
pos, err := f.pos(bitmapID, profileID)
|
||||
pos, err := f.pos(rowID, columnID)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
|
@ -426,38 +425,38 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (changed bool, err error
|
|||
}
|
||||
|
||||
// Invalidate block checksum.
|
||||
delete(f.checksums, int(bitmapID/HashBlockSize))
|
||||
delete(f.checksums, int(rowID/HashBlockSize))
|
||||
|
||||
// Increment number of operations until snapshot is required.
|
||||
if err := f.incrementOpN(); err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Get the bitmap from bitmapCache or fragment.storage.
|
||||
bm := f.bitmap(bitmapID, true, true)
|
||||
bm.ClearBit(profileID)
|
||||
// Get the row from cache or fragment.storage.
|
||||
bm := f.row(rowID, true, true)
|
||||
bm.ClearBit(columnID)
|
||||
|
||||
// Update the cache.
|
||||
f.cache.Add(bitmapID, bm.Count())
|
||||
f.cache.Add(rowID, bm.Count())
|
||||
|
||||
f.stats.Count("clearN", 1)
|
||||
|
||||
return changed, nil
|
||||
}
|
||||
|
||||
// pos translates the bitmap ID and profile ID into a position in the storage bitmap.
|
||||
func (f *Fragment) pos(bitmapID, profileID uint64) (uint64, error) {
|
||||
// Return an error if the profile ID is out of the range of the fragment's slice.
|
||||
minProfileID := f.slice * SliceWidth
|
||||
if profileID < minProfileID || profileID >= minProfileID+SliceWidth {
|
||||
return 0, errors.New("profile out of bounds")
|
||||
// pos translates the row ID and column ID into a position in the storage bitmap.
|
||||
func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
|
||||
// Return an error if the column ID is out of the range of the fragment's slice.
|
||||
minColumnID := f.slice * SliceWidth
|
||||
if columnID < minColumnID || columnID >= minColumnID+SliceWidth {
|
||||
return 0, errors.New("column out of bounds")
|
||||
}
|
||||
return Pos(bitmapID, profileID), nil
|
||||
return Pos(rowID, columnID), nil
|
||||
}
|
||||
|
||||
// ForEachBit executes fn for every bit set in the fragment.
|
||||
// Errors returned from fn are passed through.
|
||||
func (f *Fragment) ForEachBit(fn func(bitmapID, profileID uint64) error) error {
|
||||
func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
|
|
@ -474,15 +473,15 @@ func (f *Fragment) ForEachBit(fn func(bitmapID, profileID uint64) error) error {
|
|||
return err
|
||||
}
|
||||
|
||||
// Top returns the top bitmaps from the fragment.
|
||||
// If opt.Src is specified then only bitmaps which intersect src are returned.
|
||||
// If opt.FilterValues exist then the bitmap attribute specified by field is matched.
|
||||
// Top returns the top rows from the fragment.
|
||||
// If opt.Src is specified then only rows which intersect src are returned.
|
||||
// If opt.FilterValues exist then the row attribute specified by field is matched.
|
||||
func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
|
||||
// Retrieve pairs. If no bitmap ids specified then return from cache.
|
||||
pairs := f.topBitmapPairs(opt.BitmapIDs)
|
||||
// Retrieve pairs. If no row ids specified then return from cache.
|
||||
pairs := f.topBitmapPairs(opt.RowIDs)
|
||||
|
||||
// If BitmapIDs are provided, we don't want to truncate the result set
|
||||
if len(opt.BitmapIDs) > 0 {
|
||||
// If row ids are provided, we don't want to truncate the result set
|
||||
if len(opt.RowIDs) > 0 {
|
||||
opt.N = 0
|
||||
}
|
||||
|
||||
|
|
@ -509,9 +508,9 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
|
|||
// Iterate over rankings and add to results until we have enough.
|
||||
results := &PairHeap{}
|
||||
for _, pair := range pairs {
|
||||
bitmapID, cnt := pair.ID, pair.Count
|
||||
rowID, cnt := pair.ID, pair.Count
|
||||
|
||||
// Ignore empty bitmaps.
|
||||
// Ignore empty rows.
|
||||
if cnt <= 0 {
|
||||
continue
|
||||
}
|
||||
|
|
@ -531,7 +530,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
|
|||
|
||||
// Apply filter, if set.
|
||||
if filters != nil {
|
||||
attr, err := f.BitmapAttrStore.Attrs(bitmapID)
|
||||
attr, err := f.RowAttrStore.Attrs(rowID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} else if attr == nil {
|
||||
|
|
@ -548,7 +547,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
|
|||
// Calculate count and append.
|
||||
count := cnt
|
||||
if opt.Src != nil {
|
||||
count = opt.Src.IntersectionCount(f.Bitmap(bitmapID))
|
||||
count = opt.Src.IntersectionCount(f.Row(rowID))
|
||||
}
|
||||
if count == 0 {
|
||||
continue
|
||||
|
|
@ -566,7 +565,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
|
|||
}
|
||||
}
|
||||
|
||||
heap.Push(results, Pair{ID: bitmapID, Count: count})
|
||||
heap.Push(results, Pair{ID: rowID, Count: count})
|
||||
|
||||
// If we reach the requested number of pairs and we are not computing
|
||||
// intersections then simply exit. If we are intersecting then sort
|
||||
|
|
@ -584,20 +583,20 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
|
|||
// If it's too low then don't try finding anymore pairs.
|
||||
threshold := results.Pairs[0].Count
|
||||
|
||||
// If the bitmap doesn't have enough bits set before the intersection
|
||||
// then we can assume that any remaining bitmaps also have a count too low.
|
||||
// If the row doesn't have enough bits set before the intersection
|
||||
// then we can assume that any remaining rows also have a count too low.
|
||||
if threshold < opt.MinThreshold || cnt < threshold {
|
||||
break
|
||||
}
|
||||
|
||||
// Calculate the intersecting bit count and skip if it's below our
|
||||
// last bitmap in our current result set.
|
||||
count := opt.Src.IntersectionCount(f.Bitmap(bitmapID))
|
||||
// last row in our current result set.
|
||||
count := opt.Src.IntersectionCount(f.Row(rowID))
|
||||
if count < threshold {
|
||||
continue
|
||||
}
|
||||
|
||||
heap.Push(results, Pair{ID: bitmapID, Count: count})
|
||||
heap.Push(results, Pair{ID: rowID, Count: count})
|
||||
}
|
||||
|
||||
//Pop first opt.N elements out of heap
|
||||
|
|
@ -611,32 +610,32 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
|
|||
return r, nil
|
||||
}
|
||||
|
||||
func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair {
|
||||
// If no specific bitmaps are requested, retrieve top bitmaps.
|
||||
if len(bitmapIDs) == 0 {
|
||||
func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair {
|
||||
// If no specific rows are requested, retrieve top rows.
|
||||
if len(rowIDs) == 0 {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.cache.Invalidate()
|
||||
return f.cache.Top()
|
||||
}
|
||||
|
||||
// Otherwise retrieve specific bitmaps.
|
||||
pairs := make([]BitmapPair, 0, len(bitmapIDs))
|
||||
for _, bitmapID := range bitmapIDs {
|
||||
// Otherwise retrieve specific rows.
|
||||
pairs := make([]BitmapPair, 0, len(rowIDs))
|
||||
for _, rowID := range rowIDs {
|
||||
// Look up cache first, if available.
|
||||
if n := f.cache.Get(bitmapID); n > 0 {
|
||||
if n := f.cache.Get(rowID); n > 0 {
|
||||
pairs = append(pairs, BitmapPair{
|
||||
ID: bitmapID,
|
||||
ID: rowID,
|
||||
Count: n,
|
||||
})
|
||||
continue
|
||||
}
|
||||
|
||||
bm := f.Bitmap(bitmapID)
|
||||
bm := f.Row(rowID)
|
||||
if bm.Count() > 0 {
|
||||
// Otherwise load from storage.
|
||||
pairs = append(pairs, BitmapPair{
|
||||
ID: bitmapID,
|
||||
ID: rowID,
|
||||
Count: bm.Count(),
|
||||
})
|
||||
}
|
||||
|
|
@ -647,14 +646,14 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair {
|
|||
|
||||
// TopOptions represents options passed into the Top() function.
|
||||
type TopOptions struct {
|
||||
// Number of bitmaps to return.
|
||||
// Number of rows to return.
|
||||
N int
|
||||
|
||||
// Bitmap to intersect with.
|
||||
Src *Bitmap
|
||||
|
||||
// Specific bitmaps to filter against.
|
||||
BitmapIDs []uint64
|
||||
// Specific rows to filter against.
|
||||
RowIDs []uint64
|
||||
MinThreshold uint64
|
||||
|
||||
// Filter field name & values.
|
||||
|
|
@ -768,14 +767,14 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i
|
|||
}
|
||||
}
|
||||
|
||||
// BlockData returns bits in a block as bitmap & profile ID pairs.
|
||||
func (f *Fragment) BlockData(id int) (bitmapIDs, profileIDs []uint64) {
|
||||
// BlockData returns bits in a block as row & column ID pairs.
|
||||
func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
|
||||
f.storage.ForEachRange(uint64(id)*HashBlockSize*SliceWidth, (uint64(id)+1)*HashBlockSize*SliceWidth, func(i uint64) {
|
||||
bitmapIDs = append(bitmapIDs, i/SliceWidth)
|
||||
profileIDs = append(profileIDs, i%SliceWidth)
|
||||
rowIDs = append(rowIDs, i/SliceWidth)
|
||||
columnIDs = append(columnIDs, i%SliceWidth)
|
||||
})
|
||||
return
|
||||
}
|
||||
|
|
@ -789,8 +788,8 @@ func (f *Fragment) BlockData(id int) (bitmapIDs, profileIDs []uint64) {
|
|||
func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) {
|
||||
// Ensure that all pair sets are of equal length.
|
||||
for i := range data {
|
||||
if len(data[i].BitmapIDs) != len(data[i].ProfileIDs) {
|
||||
return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].BitmapIDs), len(data[i].ProfileIDs))
|
||||
if len(data[i].RowIDs) != len(data[i].ColumnIDs) {
|
||||
return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs))
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -801,22 +800,22 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
|
|||
sets = make([]PairSet, len(data)+1)
|
||||
clears = make([]PairSet, len(data)+1)
|
||||
|
||||
// Limit upper bitmap/profile pair.
|
||||
maxBitmapID := uint64(id+1) * HashBlockSize
|
||||
maxProfileID := uint64(SliceWidth)
|
||||
// Limit upper row/column pair.
|
||||
maxRowID := uint64(id+1) * HashBlockSize
|
||||
maxColumnID := uint64(SliceWidth)
|
||||
|
||||
// Create buffered iterator for local block.
|
||||
itrs := make([]*BufIterator, 1, len(data)+1)
|
||||
itrs[0] = NewBufIterator(
|
||||
NewLimitIterator(
|
||||
NewRoaringIterator(f.storage.Iterator()), maxBitmapID, maxProfileID,
|
||||
NewRoaringIterator(f.storage.Iterator()), maxRowID, maxColumnID,
|
||||
),
|
||||
)
|
||||
|
||||
// Append buffered iterators for each incoming block.
|
||||
for i := range data {
|
||||
var itr Iterator = NewSliceIterator(data[i].BitmapIDs, data[i].ProfileIDs)
|
||||
itr = NewLimitIterator(itr, maxBitmapID, maxProfileID)
|
||||
var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs)
|
||||
itr = NewLimitIterator(itr, maxRowID, maxColumnID)
|
||||
itrs = append(itrs, NewBufIterator(itr))
|
||||
}
|
||||
|
||||
|
|
@ -833,8 +832,8 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
|
|||
values := make([]bool, len(itrs))
|
||||
for {
|
||||
var min struct {
|
||||
bitmapID uint64
|
||||
profileID uint64
|
||||
rowID uint64
|
||||
columnID uint64
|
||||
}
|
||||
|
||||
// Find the lowest pair.
|
||||
|
|
@ -844,9 +843,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
|
|||
if eof { // no more data
|
||||
continue
|
||||
} else if !hasData { // first pair
|
||||
min.bitmapID, min.profileID, hasData = bid, pid, true
|
||||
} else if bid < min.bitmapID || (bid == min.bitmapID && pid < min.profileID) { // lower pair
|
||||
min.bitmapID, min.profileID = bid, pid
|
||||
min.rowID, min.columnID, hasData = bid, pid, true
|
||||
} else if bid < min.rowID || (bid == min.rowID && pid < min.columnID) { // lower pair
|
||||
min.rowID, min.columnID = bid, pid
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -860,7 +859,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
|
|||
for i, itr := range itrs {
|
||||
bid, pid, eof := itr.Next()
|
||||
|
||||
values[i] = !eof && bid == min.bitmapID && pid == min.profileID
|
||||
values[i] = !eof && bid == min.rowID && pid == min.columnID
|
||||
if values[i] {
|
||||
setN++ // set
|
||||
} else {
|
||||
|
|
@ -880,25 +879,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
|
|||
|
||||
// Append to either the set or clear diff.
|
||||
if newValue {
|
||||
sets[i].BitmapIDs = append(sets[i].BitmapIDs, min.bitmapID)
|
||||
sets[i].ProfileIDs = append(sets[i].ProfileIDs, min.profileID)
|
||||
sets[i].RowIDs = append(sets[i].RowIDs, min.rowID)
|
||||
sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID)
|
||||
} else {
|
||||
clears[i].BitmapIDs = append(sets[i].BitmapIDs, min.bitmapID)
|
||||
clears[i].ProfileIDs = append(sets[i].ProfileIDs, min.profileID)
|
||||
clears[i].RowIDs = append(sets[i].RowIDs, min.rowID)
|
||||
clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Set local bits.
|
||||
for i := range sets[0].ProfileIDs {
|
||||
if _, err := f.setBit(sets[0].BitmapIDs[i], (f.Slice()*SliceWidth)+sets[0].ProfileIDs[i]); err != nil {
|
||||
for i := range sets[0].ColumnIDs {
|
||||
if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Clear local bits.
|
||||
for i := range clears[0].ProfileIDs {
|
||||
if _, err := f.clearBit(clears[0].BitmapIDs[i], (f.Slice()*SliceWidth)+clears[0].ProfileIDs[i]); err != nil {
|
||||
for i := range clears[0].ColumnIDs {
|
||||
if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
|
@ -908,12 +907,12 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
|
|||
|
||||
// Import bulk imports a set of bits and then snapshots the storage.
|
||||
// This does not affect the fragment's cache.
|
||||
func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
|
||||
func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
// Verify that there are an equal number of bitmap ids and profile ids.
|
||||
if len(bitmapIDs) != len(profileIDs) {
|
||||
return fmt.Errorf("mismatch of bitmap/profile len: %d != %d", len(bitmapIDs), len(profileIDs))
|
||||
// Verify that there are an equal number of row ids and column ids.
|
||||
if len(rowIDs) != len(columnIDs) {
|
||||
return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs))
|
||||
}
|
||||
|
||||
// Disconnect op writer so we don't append updates.
|
||||
|
|
@ -924,11 +923,11 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
|
|||
lastID := uint64(0)
|
||||
if err := func() error {
|
||||
set := make(map[uint64]struct{})
|
||||
for i := range bitmapIDs {
|
||||
bitmapID, profileID := bitmapIDs[i], profileIDs[i]
|
||||
for i := range rowIDs {
|
||||
rowID, columnID := rowIDs[i], columnIDs[i]
|
||||
|
||||
// Determine the position of the bit in the storage.
|
||||
pos, err := f.pos(bitmapID, profileID)
|
||||
pos, err := f.pos(rowID, columnID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -938,25 +937,25 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
f.stats.Count("ImportBit", 1)
|
||||
// import optimization to avoid linear foreach calls
|
||||
// slight risk of concurrent cache counter being off but
|
||||
// no real danger
|
||||
if i == 0 || bitmapID != lastID {
|
||||
lastID = bitmapID
|
||||
set[bitmapID] = struct{}{}
|
||||
if i == 0 || rowID != lastID {
|
||||
lastID = rowID
|
||||
set[rowID] = struct{}{}
|
||||
}
|
||||
|
||||
// Invalidate block checksum.
|
||||
delete(f.checksums, int(bitmapID/HashBlockSize))
|
||||
delete(f.checksums, int(rowID/HashBlockSize))
|
||||
}
|
||||
|
||||
// Update cache counts for all bitmaps.
|
||||
for bitmapID := range set {
|
||||
// Import should ALWAYS have bitmap() load a new bm from fragment.storage
|
||||
// because the bitmap that's in bitmapCache hasn't been updated with
|
||||
// Update cache counts for all rows.
|
||||
for rowID := range set {
|
||||
// Import should ALWAYS have row() load a new bm from fragment.storage
|
||||
// because the row that's in rowCache hasn't been updated with
|
||||
// this import's data.
|
||||
f.cache.BulkAdd(bitmapID, f.bitmap(bitmapID, false, false).Count())
|
||||
f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count())
|
||||
}
|
||||
|
||||
f.cache.Invalidate()
|
||||
|
|
@ -995,15 +994,18 @@ func (f *Fragment) Snapshot() error {
|
|||
defer f.mu.Unlock()
|
||||
return f.snapshot()
|
||||
}
|
||||
func track(start time.Time, name string, logger *log.Logger) {
|
||||
func track(start time.Time, message string, stats StatsClient, logger *log.Logger) {
|
||||
elapsed := time.Since(start)
|
||||
logger.Printf("%s took %s", name, elapsed)
|
||||
logger.Printf("%s took %s", message, elapsed)
|
||||
stats.Histogram("snapshot", elapsed.Seconds())
|
||||
}
|
||||
|
||||
func (f *Fragment) snapshot() error {
|
||||
logger := f.logger()
|
||||
logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.db, f.frame, f.view, f.slice)
|
||||
defer track(time.Now(), fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.db, f.frame, f.view, f.slice), logger)
|
||||
logger.Printf("fragment: snapshotting %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
|
||||
completeMessage := fmt.Sprintf("fragment: snapshot complete %s/%s/%s/%d", f.index, f.frame, f.view, f.slice)
|
||||
start := time.Now()
|
||||
defer track(start, completeMessage, f.stats, logger)
|
||||
|
||||
// Create a temporary file to snapshot to.
|
||||
snapshotPath := f.path + SnapshotExt
|
||||
|
|
@ -1061,13 +1063,11 @@ func (f *Fragment) flushCache() error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// Retrieve a list of bitmap ids from the cache.
|
||||
bitmapIDs := f.cache.BitmapIDs()
|
||||
// Retrieve a list of row ids from the cache.
|
||||
ids := f.cache.IDs()
|
||||
|
||||
// Marshal cache data to bytes.
|
||||
buf, err := proto.Marshal(&internal.Cache{
|
||||
BitmapIDs: bitmapIDs,
|
||||
})
|
||||
buf, err := proto.Marshal(&internal.Cache{IDs: ids})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -1253,7 +1253,7 @@ func (f *Fragment) readCacheFromArchive(r io.Reader) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// FragmentBlock represents info about a subsection of the bitmaps in a block.
|
||||
// FragmentBlock represents info about a subsection of the rows in a block.
|
||||
// This is used for comparing data in remote blocks for active anti-entropy.
|
||||
type FragmentBlock struct {
|
||||
ID int `json:"id"`
|
||||
|
|
@ -1309,7 +1309,7 @@ func (s *FragmentSyncer) isClosing() bool {
|
|||
// then merges any blocks which have differences.
|
||||
func (s *FragmentSyncer) SyncFragment() error {
|
||||
// Determine replica set.
|
||||
nodes := s.Cluster.FragmentNodes(s.Fragment.DB(), s.Fragment.Slice())
|
||||
nodes := s.Cluster.FragmentNodes(s.Fragment.Index(), s.Fragment.Slice())
|
||||
if len(nodes) == 1 {
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1329,7 +1329,7 @@ func (s *FragmentSyncer) SyncFragment() error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.DB(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
|
||||
blocks, err := client.FragmentBlocks(context.Background(), s.Fragment.Index(), s.Fragment.Frame(), s.Fragment.View(), s.Fragment.Slice())
|
||||
if err != nil && err != ErrFragmentNotFound {
|
||||
return err
|
||||
}
|
||||
|
|
@ -1376,17 +1376,17 @@ func (s *FragmentSyncer) SyncFragment() error {
|
|||
if byteSlicesEqual(checksums) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Synchronize block.
|
||||
if err := s.syncBlock(blockID); err != nil {
|
||||
return fmt.Errorf("sync block: id=%d, err=%s", blockID, err)
|
||||
}
|
||||
s.Fragment.stats.Count("BlockRepair", 1)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncBlock sends and receives all bitmaps for a given block.
|
||||
// syncBlock sends and receives all rows for a given block.
|
||||
// Returns an error if any remote hosts are unreachable.
|
||||
func (s *FragmentSyncer) syncBlock(id int) error {
|
||||
f := s.Fragment
|
||||
|
|
@ -1394,7 +1394,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
|
|||
// Read pairs from each remote block.
|
||||
var pairSets []PairSet
|
||||
var clients []*Client
|
||||
for _, node := range s.Cluster.FragmentNodes(f.DB(), f.Slice()) {
|
||||
for _, node := range s.Cluster.FragmentNodes(f.Index(), f.Slice()) {
|
||||
if s.Host == node.Host {
|
||||
continue
|
||||
}
|
||||
|
|
@ -1411,14 +1411,14 @@ func (s *FragmentSyncer) syncBlock(id int) error {
|
|||
clients = append(clients, client)
|
||||
|
||||
// Only sync the standard block.
|
||||
bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id)
|
||||
rowIDs, columnIDs, err := client.BlockData(context.Background(), f.Index(), f.Frame(), ViewStandard, f.Slice(), id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
pairSets = append(pairSets, PairSet{
|
||||
ProfileIDs: profileIDs,
|
||||
BitmapIDs: bitmapIDs,
|
||||
ColumnIDs: columnIDs,
|
||||
RowIDs: rowIDs,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -1438,7 +1438,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
|
|||
set, clear := sets[i], clears[i]
|
||||
|
||||
// Ignore if there are no differences.
|
||||
if len(set.ProfileIDs) == 0 && len(clear.ProfileIDs) == 0 {
|
||||
if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
|
|
@ -1446,11 +1446,11 @@ func (s *FragmentSyncer) syncBlock(id int) error {
|
|||
var buf bytes.Buffer
|
||||
|
||||
// Only sync the standard block.
|
||||
for j := 0; j < len(set.ProfileIDs); j++ {
|
||||
fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), set.BitmapIDs[j], (f.Slice()*SliceWidth)+set.ProfileIDs[j])
|
||||
for j := 0; j < len(set.ColumnIDs); j++ {
|
||||
fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
|
||||
}
|
||||
for j := 0; j < len(clear.ProfileIDs); j++ {
|
||||
fmt.Fprintf(&buf, "ClearBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), clear.BitmapIDs[j], (f.Slice()*SliceWidth)+clear.ProfileIDs[j])
|
||||
for j := 0; j < len(clear.ColumnIDs); j++ {
|
||||
fmt.Fprintf(&buf, "ClearBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
|
||||
}
|
||||
|
||||
// Verify sync is not prematurely closing.
|
||||
|
|
@ -1459,7 +1459,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
|
|||
}
|
||||
|
||||
// Execute query.
|
||||
_, err := clients[i].ExecuteQuery(context.Background(), f.DB(), buf.String(), false)
|
||||
_, err := clients[i].ExecuteQuery(context.Background(), f.Index(), buf.String(), false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -1476,10 +1476,10 @@ func madvise(b []byte, advice int) (err error) {
|
|||
return
|
||||
}
|
||||
|
||||
// PairSet is a list of equal length bitmap and profile id lists.
|
||||
// PairSet is a list of equal length row and column id lists.
|
||||
type PairSet struct {
|
||||
BitmapIDs []uint64
|
||||
ProfileIDs []uint64
|
||||
RowIDs []uint64
|
||||
ColumnIDs []uint64
|
||||
}
|
||||
|
||||
// byteSlicesEqual returns true if all slices are equal.
|
||||
|
|
@ -1496,7 +1496,7 @@ func byteSlicesEqual(a [][]byte) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
// Pos returns the bitmap position of a bitmap/profile pair.
|
||||
func Pos(bitmapID, profileID uint64) uint64 {
|
||||
return (bitmapID * SliceWidth) + (profileID % SliceWidth)
|
||||
// Pos returns the row position of a row/column pair.
|
||||
func Pos(rowID, columnID uint64) uint64 {
|
||||
return (rowID * SliceWidth) + (columnID % SliceWidth)
|
||||
}
|
||||
|
|
|
|||
237
fragment_test.go
237
fragment_test.go
|
|
@ -23,7 +23,7 @@ const SliceWidth = pilosa.SliceWidth
|
|||
|
||||
// Ensure a fragment can set a bit and retrieve it.
|
||||
func TestFragment_SetBit(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Set bits on the fragment.
|
||||
|
|
@ -35,26 +35,26 @@ func TestFragment_SetBit(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify counts on bitmaps.
|
||||
if n := f.Bitmap(120).Count(); n != 2 {
|
||||
// Verify counts on rows.
|
||||
if n := f.Row(120).Count(); n != 2 {
|
||||
t.Fatalf("unexpected count: %d", n)
|
||||
} else if n := f.Bitmap(121).Count(); n != 1 {
|
||||
} else if n := f.Row(121).Count(); n != 1 {
|
||||
t.Fatalf("unexpected count: %d", n)
|
||||
}
|
||||
|
||||
// Close and reopen the fragment & verify the data.
|
||||
if err := f.Reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if n := f.Bitmap(120).Count(); n != 2 {
|
||||
} else if n := f.Row(120).Count(); n != 2 {
|
||||
t.Fatalf("unexpected count (reopen): %d", n)
|
||||
} else if n := f.Bitmap(121).Count(); n != 1 {
|
||||
} else if n := f.Row(121).Count(); n != 1 {
|
||||
t.Fatalf("unexpected count (reopen): %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can clear a set bit.
|
||||
func TestFragment_ClearBit(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Set and then clear bits on the fragment.
|
||||
|
|
@ -66,22 +66,22 @@ func TestFragment_ClearBit(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify count on bitmap.
|
||||
if n := f.Bitmap(1000).Count(); n != 1 {
|
||||
// Verify count on row.
|
||||
if n := f.Row(1000).Count(); n != 1 {
|
||||
t.Fatalf("unexpected count: %d", n)
|
||||
}
|
||||
|
||||
// Close and reopen the fragment & verify the data.
|
||||
if err := f.Reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if n := f.Bitmap(1000).Count(); n != 1 {
|
||||
} else if n := f.Row(1000).Count(); n != 1 {
|
||||
t.Fatalf("unexpected count (reopen): %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can snapshot correctly.
|
||||
func TestFragment_Snapshot(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Set and then clear bits on the fragment.
|
||||
|
|
@ -96,21 +96,21 @@ func TestFragment_Snapshot(t *testing.T) {
|
|||
// Snapshot bitmap and verify data.
|
||||
if err := f.Snapshot(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if n := f.Bitmap(1000).Count(); n != 1 {
|
||||
} else if n := f.Row(1000).Count(); n != 1 {
|
||||
t.Fatalf("unexpected count: %d", n)
|
||||
}
|
||||
|
||||
// Close and reopen the fragment & verify the data.
|
||||
if err := f.Reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if n := f.Bitmap(1000).Count(); n != 1 {
|
||||
} else if n := f.Row(1000).Count(); n != 1 {
|
||||
t.Fatalf("unexpected count (reopen): %d", n)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can iterate over all bits in order.
|
||||
func TestFragment_ForEachBit(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Set bits on the fragment.
|
||||
|
|
@ -124,8 +124,8 @@ func TestFragment_ForEachBit(t *testing.T) {
|
|||
|
||||
// Iterate over bits.
|
||||
var result [][2]uint64
|
||||
if err := f.ForEachBit(func(bitmapID, profileID uint64) error {
|
||||
result = append(result, [2]uint64{bitmapID, profileID})
|
||||
if err := f.ForEachBit(func(rowID, columnID uint64) error {
|
||||
result = append(result, [2]uint64{rowID, columnID})
|
||||
return nil
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
|
|
@ -139,15 +139,15 @@ func TestFragment_ForEachBit(t *testing.T) {
|
|||
|
||||
// Ensure a fragment can return the top n results.
|
||||
func TestFragment_Top(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Set bits on the bitmaps 100, 101, & 102.
|
||||
// Set bits on the rows 100, 101, & 102.
|
||||
f.MustSetBits(100, 1, 3, 200)
|
||||
f.MustSetBits(101, 1)
|
||||
f.MustSetBits(102, 1, 2)
|
||||
|
||||
// Retrieve top bitmaps.
|
||||
// Retrieve top rows.
|
||||
if pairs, err := f.Top(pilosa.TopOptions{N: 2}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if len(pairs) != 2 {
|
||||
|
|
@ -159,21 +159,21 @@ func TestFragment_Top(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can filter bitmaps when retrieving the top n bitmaps.
|
||||
// Ensure a fragment can filter rows when retrieving the top n rows.
|
||||
func TestFragment_Top_Filter(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Set bits on the bitmaps 100, 101, & 102.
|
||||
// Set bits on the rows 100, 101, & 102.
|
||||
f.MustSetBits(100, 1, 3, 200)
|
||||
f.MustSetBits(101, 1)
|
||||
f.MustSetBits(102, 1, 2)
|
||||
|
||||
// Assign attributes.
|
||||
f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)})
|
||||
f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)})
|
||||
f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)})
|
||||
f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)})
|
||||
|
||||
// Retrieve top bitmaps.
|
||||
// Retrieve top rows.
|
||||
if pairs, err := f.Top(pilosa.TopOptions{
|
||||
N: 2,
|
||||
FilterField: "x",
|
||||
|
|
@ -189,21 +189,21 @@ func TestFragment_Top_Filter(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can return top bitmaps that intersect with an input bitmap.
|
||||
// Ensure a fragment can return top rows that intersect with an input row.
|
||||
func TestFragment_TopN_Intersect(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Create an intersecting input bitmap.
|
||||
// Create an intersecting input row.
|
||||
src := pilosa.NewBitmap(1, 2, 3)
|
||||
|
||||
// Set bits on various bitmaps.
|
||||
// Set bits on various rows.
|
||||
f.MustSetBits(100, 1, 10, 11, 12) // one intersection
|
||||
f.MustSetBits(101, 1, 2, 3, 4) // three intersections
|
||||
f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections
|
||||
f.MustSetBits(103, 1000, 1001, 1002) // no intersection
|
||||
|
||||
// Retrieve top bitmaps.
|
||||
// Retrieve top rows.
|
||||
if pairs, err := f.Top(pilosa.TopOptions{N: 3, Src: src}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
|
|
@ -215,29 +215,29 @@ func TestFragment_TopN_Intersect(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can return top bitmaps that have many bits set.
|
||||
// Ensure a fragment can return top rows that have many bits set.
|
||||
func TestFragment_TopN_Intersect_Large(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("short mode")
|
||||
}
|
||||
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Create an intersecting input bitmap.
|
||||
// Create an intersecting input row.
|
||||
src := pilosa.NewBitmap(
|
||||
980, 981, 982, 983, 984, 985, 986, 987, 988, 989,
|
||||
990, 991, 992, 993, 994, 995, 996, 997, 998, 999,
|
||||
)
|
||||
|
||||
// Set bits on bitmaps 0 - 999. Higher bitmaps have higher bit counts.
|
||||
// Set bits on rows 0 - 999. Higher rows have higher bit counts.
|
||||
for i := uint64(0); i < 1000; i++ {
|
||||
for j := uint64(0); j < i; j++ {
|
||||
f.MustSetBits(i, j)
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieve top bitmaps.
|
||||
// Retrieve top rows.
|
||||
if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
|
|
@ -256,18 +256,18 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure a fragment can return top bitmaps when specified by ID.
|
||||
func TestFragment_TopN_BitmapIDs(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
// Ensure a fragment can return top rows when specified by ID.
|
||||
func TestFragment_TopN_IDs(t *testing.T) {
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Set bits on various bitmaps.
|
||||
// Set bits on various rows.
|
||||
f.MustSetBits(100, 1, 2, 3)
|
||||
f.MustSetBits(101, 4, 5, 6, 7)
|
||||
f.MustSetBits(102, 8, 9, 10, 11, 12)
|
||||
|
||||
// Retrieve top bitmaps.
|
||||
if pairs, err := f.Top(pilosa.TopOptions{BitmapIDs: []uint64{100, 101, 200}}); err != nil {
|
||||
// Retrieve top rows.
|
||||
if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
|
||||
{ID: 101, Count: 4},
|
||||
|
|
@ -282,12 +282,12 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
|
|||
slice := uint64(0)
|
||||
cacheSize := uint32(3)
|
||||
|
||||
// Create DB.
|
||||
db := MustOpenDB()
|
||||
defer db.Close()
|
||||
// Create Index.
|
||||
index := MustOpenIndex()
|
||||
defer index.Close()
|
||||
|
||||
// Create frame.
|
||||
frame, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize})
|
||||
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: cacheSize})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -307,16 +307,16 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
|
|||
frag.Close()
|
||||
|
||||
f := &Fragment{
|
||||
Fragment: frag,
|
||||
BitmapAttrStore: MustOpenAttrStore(),
|
||||
Fragment: frag,
|
||||
RowAttrStore: MustOpenAttrStore(),
|
||||
}
|
||||
f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
|
||||
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
|
||||
if err := f.Open(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// Set bits on various bitmaps.
|
||||
// Set bits on various rows.
|
||||
f.MustSetBits(100, 1, 2, 3)
|
||||
f.MustSetBits(101, 4, 5, 6, 7)
|
||||
f.MustSetBits(102, 8, 9, 10, 11, 12)
|
||||
|
|
@ -332,7 +332,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
|
|||
{ID: 102, Count: 5},
|
||||
}
|
||||
|
||||
// Retrieve top bitmaps.
|
||||
// Retrieve top rows.
|
||||
if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if len(pairs) > int(cacheSize) {
|
||||
|
|
@ -346,7 +346,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
|
|||
|
||||
// Ensure fragment can return a checksum for its blocks.
|
||||
func TestFragment_Checksum(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Retrieve checksum and set bits.
|
||||
|
|
@ -365,7 +365,7 @@ func TestFragment_Checksum(t *testing.T) {
|
|||
|
||||
// Ensure fragment can return a checksum for a given block.
|
||||
func TestFragment_Blocks(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Retrieve initial checksum.
|
||||
|
|
@ -381,7 +381,7 @@ func TestFragment_Blocks(t *testing.T) {
|
|||
}
|
||||
prev = blocks
|
||||
|
||||
// Set bit on different bitmap.
|
||||
// Set bit on different row.
|
||||
if _, err := f.SetBit(20, 0); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -391,7 +391,7 @@ func TestFragment_Blocks(t *testing.T) {
|
|||
}
|
||||
prev = blocks
|
||||
|
||||
// Set bit on different profile.
|
||||
// Set bit on different column.
|
||||
if _, err := f.SetBit(20, 100); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -403,7 +403,7 @@ func TestFragment_Blocks(t *testing.T) {
|
|||
|
||||
// Ensure fragment returns an empty checksum if no data exists for a block.
|
||||
func TestFragment_Blocks_Empty(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Set bits on a different block.
|
||||
|
|
@ -421,7 +421,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
|
|||
|
||||
// Ensure a fragment's cache can be persisted between restarts.
|
||||
func TestFragment_LRUCache_Persistence(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Set bits on the fragment.
|
||||
|
|
@ -453,11 +453,11 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
|
|||
|
||||
// Ensure a fragment's cache can be persisted between restarts.
|
||||
func TestFragment_RankCache_Persistence(t *testing.T) {
|
||||
db := MustOpenDB()
|
||||
defer db.Close()
|
||||
index := MustOpenIndex()
|
||||
defer index.Close()
|
||||
|
||||
// Create frame.
|
||||
frame, err := db.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
|
||||
frame, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -488,13 +488,13 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
|
|||
t.Fatalf("unexpected cache len: %d", cache.Len())
|
||||
}
|
||||
|
||||
// Reopen the database.
|
||||
if err := db.Reopen(); err != nil {
|
||||
// Reopen the index.
|
||||
if err := index.Reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Re-fetch fragment.
|
||||
f = db.Frame("f").View(pilosa.ViewStandard).Fragment(0)
|
||||
f = index.Frame("f").View(pilosa.ViewStandard).Fragment(0)
|
||||
|
||||
// Re-verify correct cache type and size.
|
||||
if cache, ok := f.Cache().(*pilosa.RankCache); !ok {
|
||||
|
|
@ -506,7 +506,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
|
|||
|
||||
// Ensure a fragment can be copied to another fragment.
|
||||
func TestFragment_WriteTo_ReadFrom(t *testing.T) {
|
||||
f0 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f0 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f0.Close()
|
||||
|
||||
// Set and then clear bits on the fragment.
|
||||
|
|
@ -531,7 +531,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
|
|||
}
|
||||
|
||||
// Read into another fragment.
|
||||
f1 := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f1 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
if rn, err := f1.ReadFrom(&buf); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if wn != rn {
|
||||
|
|
@ -544,7 +544,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
|
|||
}
|
||||
|
||||
// Verify data in other fragment.
|
||||
if a := f1.Bitmap(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
|
||||
if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
|
||||
t.Fatalf("unexpected bits: %+v", a)
|
||||
}
|
||||
|
||||
|
|
@ -553,47 +553,18 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
} else if n := f1.Cache().Len(); n != 1 {
|
||||
t.Fatalf("unexpected cache size (reopen): %d", n)
|
||||
} else if a := f1.Bitmap(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
|
||||
} else if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
|
||||
t.Fatalf("unexpected bits (reopen): %+v", a)
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
func BenchmarkFragment_BlockChecksum_Fill1(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.01) }
|
||||
func BenchmarkFragment_BlockChecksum_Fill10(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.10) }
|
||||
func BenchmarkFragment_BlockChecksum_Fill50(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.50) }
|
||||
|
||||
func benchmarkFragmentBlockChecksum(b *testing.B, fillPercent float64) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
// Fill fragment.
|
||||
bitmapIDs, profileIDs := GenerateImportFill(pilosa.HashBlockSize, fillPercent)
|
||||
if err := f.Import(bitmapIDs, profileIDs); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
|
||||
// Calculate block checksum.
|
||||
for i := 0; i < b.N; i++ {
|
||||
f.InvalidateChecksums()
|
||||
|
||||
if chksum := f.BlockChecksum(0); chksum == nil {
|
||||
b.Fatal("expected checksum")
|
||||
}
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
func BenchmarkFragment_Blocks(b *testing.B) {
|
||||
if *FragmentPath == "" {
|
||||
b.Skip("no fragment specified")
|
||||
}
|
||||
|
||||
// Open the fragment specified by the path.
|
||||
f := pilosa.NewFragment(*FragmentPath, "d", "f", pilosa.ViewStandard, 0)
|
||||
f := pilosa.NewFragment(*FragmentPath, "i", "f", pilosa.ViewStandard, 0)
|
||||
if err := f.Open(); err != nil {
|
||||
b.Fatal(err)
|
||||
}
|
||||
|
|
@ -609,7 +580,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
|
|||
}
|
||||
|
||||
func BenchmarkFragment_IntersectionCount(b *testing.B) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
f.MaxOpN = math.MaxInt32
|
||||
|
||||
|
|
@ -633,7 +604,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
|
|||
// Start benchmark
|
||||
b.ResetTimer()
|
||||
for i := 0; i < b.N; i++ {
|
||||
if n := f.Bitmap(1).IntersectionCount(f.Bitmap(2)); n == 0 {
|
||||
if n := f.Row(1).IntersectionCount(f.Row(2)); n == 0 {
|
||||
b.Fatalf("unexpected count: %d", n)
|
||||
}
|
||||
}
|
||||
|
|
@ -642,11 +613,11 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
|
|||
// Fragment is a test wrapper for pilosa.Fragment.
|
||||
type Fragment struct {
|
||||
*pilosa.Fragment
|
||||
BitmapAttrStore *AttrStore
|
||||
RowAttrStore *AttrStore
|
||||
}
|
||||
|
||||
// NewFragment returns a new instance of Fragment with a temporary path.
|
||||
func NewFragment(db, frame, view string, slice uint64) *Fragment {
|
||||
func NewFragment(index, frame, view string, slice uint64) *Fragment {
|
||||
file, err := ioutil.TempFile("", "pilosa-fragment-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -654,16 +625,16 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment {
|
|||
file.Close()
|
||||
|
||||
f := &Fragment{
|
||||
Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice),
|
||||
BitmapAttrStore: MustOpenAttrStore(),
|
||||
Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice),
|
||||
RowAttrStore: MustOpenAttrStore(),
|
||||
}
|
||||
f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
|
||||
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
|
||||
return f
|
||||
}
|
||||
|
||||
// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error.
|
||||
func MustOpenFragment(db, frame, view string, slice uint64) *Fragment {
|
||||
f := NewFragment(db, frame, view, slice)
|
||||
func MustOpenFragment(index, frame, view string, slice uint64) *Fragment {
|
||||
f := NewFragment(index, frame, view, slice)
|
||||
if err := f.Open(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
@ -674,7 +645,7 @@ func MustOpenFragment(db, frame, view string, slice uint64) *Fragment {
|
|||
func (f *Fragment) Close() error {
|
||||
defer os.Remove(f.Path())
|
||||
defer os.Remove(f.CachePath())
|
||||
defer f.BitmapAttrStore.Close()
|
||||
defer f.RowAttrStore.Close()
|
||||
return f.Fragment.Close()
|
||||
}
|
||||
|
||||
|
|
@ -685,76 +656,76 @@ func (f *Fragment) Reopen() error {
|
|||
return err
|
||||
}
|
||||
|
||||
f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice())
|
||||
f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
|
||||
f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice())
|
||||
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
|
||||
if err := f.Open(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustSetBits sets bits on a bitmap. Panic on error.
|
||||
// MustSetBits sets bits on a row. Panic on error.
|
||||
// This function does not accept a timestamp or quantum.
|
||||
func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) {
|
||||
for _, profileID := range profileIDs {
|
||||
if _, err := f.SetBit(bitmapID, profileID); err != nil {
|
||||
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
|
||||
for _, columnID := range columnIDs {
|
||||
if _, err := f.SetBit(rowID, columnID); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MustClearBits clears bits on a bitmap. Panic on error.
|
||||
func (f *Fragment) MustClearBits(bitmapID uint64, profileIDs ...uint64) {
|
||||
for _, profileID := range profileIDs {
|
||||
if _, err := f.ClearBit(bitmapID, profileID); err != nil {
|
||||
// MustClearBits clears bits on a row. Panic on error.
|
||||
func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) {
|
||||
for _, columnID := range columnIDs {
|
||||
if _, err := f.ClearBit(rowID, columnID); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// BitmapAttrStore provides simple storage for attributes.
|
||||
type BitmapAttrStore struct {
|
||||
// RowAttrStore provides simple storage for attributes.
|
||||
type RowAttrStore struct {
|
||||
attrs map[uint64]map[string]interface{}
|
||||
}
|
||||
|
||||
// NewBitmapAttrStore returns a new instance of BitmapAttrStore.
|
||||
func NewBitmapAttrStore() *BitmapAttrStore {
|
||||
return &BitmapAttrStore{
|
||||
// NewRowAttrStore returns a new instance of RowAttrStore.
|
||||
func NewRowAttrStore() *RowAttrStore {
|
||||
return &RowAttrStore{
|
||||
attrs: make(map[uint64]map[string]interface{}),
|
||||
}
|
||||
}
|
||||
|
||||
// BitmapAttrs returns the attributes set to a bitmap id.
|
||||
func (s *BitmapAttrStore) BitmapAttrs(id uint64) (map[string]interface{}, error) {
|
||||
// RowAttrs returns the attributes set to a row id.
|
||||
func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) {
|
||||
return s.attrs[id], nil
|
||||
}
|
||||
|
||||
// SetBitmapAttrs assigns a set of attributes to a bitmap id.
|
||||
func (s *BitmapAttrStore) SetBitmapAttrs(id uint64, m map[string]interface{}) {
|
||||
// SetRowAttrs assigns a set of attributes to a row id.
|
||||
func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) {
|
||||
s.attrs[id] = m
|
||||
}
|
||||
|
||||
// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk.
|
||||
func GenerateImportFill(bitmapN int, pct float64) (bitmapIDs, profileIDs []uint64) {
|
||||
func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
|
||||
ipct := int(pct * 100)
|
||||
for i := 0; i < SliceWidth*bitmapN; i++ {
|
||||
for i := 0; i < SliceWidth*rowN; i++ {
|
||||
if i%100 >= ipct {
|
||||
continue
|
||||
}
|
||||
|
||||
bitmapIDs = append(bitmapIDs, uint64(i%SliceWidth))
|
||||
profileIDs = append(profileIDs, uint64(i/SliceWidth))
|
||||
rowIDs = append(rowIDs, uint64(i%SliceWidth))
|
||||
columnIDs = append(columnIDs, uint64(i/SliceWidth))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func TestFragment_Tanimoto(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
src := pilosa.NewBitmap(1, 2, 3)
|
||||
|
||||
// Set bits on the bitmaps 100, 101, & 102.
|
||||
// Set bits on the rows 100, 101, & 102.
|
||||
f.MustSetBits(100, 1, 3, 2, 200)
|
||||
f.MustSetBits(101, 1, 3)
|
||||
f.MustSetBits(102, 1, 2, 10, 12)
|
||||
|
|
@ -771,12 +742,12 @@ func TestFragment_Tanimoto(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestFragment_Zero_Tanimoto(t *testing.T) {
|
||||
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
|
||||
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0)
|
||||
defer f.Close()
|
||||
|
||||
src := pilosa.NewBitmap(1, 2, 3)
|
||||
|
||||
// Set bits on the bitmaps 100, 101, & 102.
|
||||
// Set bits on the rows 100, 101, & 102.
|
||||
f.MustSetBits(100, 1, 3, 2, 200)
|
||||
f.MustSetBits(101, 1, 3)
|
||||
f.MustSetBits(102, 1, 2, 10, 12)
|
||||
|
|
|
|||
122
frame.go
122
frame.go
|
|
@ -29,17 +29,17 @@ const (
|
|||
type Frame struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
db string
|
||||
index string
|
||||
name string
|
||||
timeQuantum TimeQuantum
|
||||
|
||||
views map[string]*View
|
||||
|
||||
// Bitmap attribute storage and cache
|
||||
bitmapAttrStore *AttrStore
|
||||
// Row attribute storage and cache
|
||||
rowAttrStore *AttrStore
|
||||
|
||||
broadcaster Broadcaster
|
||||
stats StatsClient
|
||||
Stats StatsClient
|
||||
|
||||
// Frame settings.
|
||||
rowLabel string
|
||||
|
|
@ -53,21 +53,21 @@ type Frame struct {
|
|||
}
|
||||
|
||||
// NewFrame returns a new instance of frame.
|
||||
func NewFrame(path, db, name string) (*Frame, error) {
|
||||
func NewFrame(path, index, name string) (*Frame, error) {
|
||||
err := ValidateName(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Frame{
|
||||
path: path,
|
||||
db: db,
|
||||
name: name,
|
||||
path: path,
|
||||
index: index,
|
||||
name: name,
|
||||
|
||||
views: make(map[string]*View),
|
||||
bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")),
|
||||
views: make(map[string]*View),
|
||||
rowAttrStore: NewAttrStore(filepath.Join(path, ".data")),
|
||||
|
||||
stats: NopStatsClient,
|
||||
Stats: NopStatsClient,
|
||||
|
||||
rowLabel: DefaultRowLabel,
|
||||
inverseEnabled: DefaultInverseEnabled,
|
||||
|
|
@ -81,14 +81,14 @@ func NewFrame(path, db, name string) (*Frame, error) {
|
|||
// Name returns the name the frame was initialized with.
|
||||
func (f *Frame) Name() string { return f.name }
|
||||
|
||||
// DB returns the database name the frame was initialized with.
|
||||
func (f *Frame) DB() string { return f.db }
|
||||
// Index returns the index name the frame was initialized with.
|
||||
func (f *Frame) Index() string { return f.index }
|
||||
|
||||
// Path returns the path the frame was initialized with.
|
||||
func (f *Frame) Path() string { return f.path }
|
||||
|
||||
// BitmapAttrStore returns the attribute storage.
|
||||
func (f *Frame) BitmapAttrStore() *AttrStore { return f.bitmapAttrStore }
|
||||
// RowAttrStore returns the attribute storage.
|
||||
func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore }
|
||||
|
||||
// MaxSlice returns the max slice in the frame.
|
||||
func (f *Frame) MaxSlice() uint64 {
|
||||
|
|
@ -215,7 +215,7 @@ func (f *Frame) Open() error {
|
|||
return err
|
||||
}
|
||||
|
||||
if err := f.bitmapAttrStore.Open(); err != nil {
|
||||
if err := f.rowAttrStore.Open(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -253,10 +253,10 @@ func (f *Frame) openViews() error {
|
|||
if err := view.Open(); err != nil {
|
||||
return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err)
|
||||
}
|
||||
view.BitmapAttrStore = f.bitmapAttrStore
|
||||
view.RowAttrStore = f.rowAttrStore
|
||||
f.views[view.Name()] = view
|
||||
|
||||
f.stats.Count("maxSlice", 1)
|
||||
f.Stats.Count("maxSlice", 1)
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -302,11 +302,11 @@ func (f *Frame) loadMeta() error {
|
|||
func (f *Frame) saveMeta() error {
|
||||
// Marshal metadata.
|
||||
buf, err := proto.Marshal(&internal.FrameMeta{
|
||||
TimeQuantum: string(f.timeQuantum),
|
||||
RowLabel: f.rowLabel,
|
||||
CacheType: f.cacheType,
|
||||
InverseEnabled: f.inverseEnabled,
|
||||
CacheType: f.cacheType,
|
||||
CacheSize: f.cacheSize,
|
||||
TimeQuantum: string(f.timeQuantum),
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -326,8 +326,8 @@ func (f *Frame) Close() error {
|
|||
defer f.mu.Unlock()
|
||||
|
||||
// Close the attribute store.
|
||||
if f.bitmapAttrStore != nil {
|
||||
_ = f.bitmapAttrStore.Close()
|
||||
if f.rowAttrStore != nil {
|
||||
_ = f.rowAttrStore.Close()
|
||||
}
|
||||
|
||||
// Close all views.
|
||||
|
|
@ -411,18 +411,18 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
|
|||
if err := view.Open(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
view.BitmapAttrStore = f.bitmapAttrStore
|
||||
view.RowAttrStore = f.rowAttrStore
|
||||
f.views[view.Name()] = view
|
||||
|
||||
return view, nil
|
||||
}
|
||||
|
||||
func (f *Frame) newView(path, name string) *View {
|
||||
view := NewView(path, f.db, f.name, name, f.cacheSize)
|
||||
view := NewView(path, f.index, f.name, name, f.cacheSize)
|
||||
view.cacheType = f.cacheType
|
||||
view.LogOutput = f.LogOutput
|
||||
view.BitmapAttrStore = f.bitmapAttrStore
|
||||
view.stats = f.stats.WithTags(fmt.Sprintf("slice:%s", name))
|
||||
view.RowAttrStore = f.rowAttrStore
|
||||
view.stats = f.Stats.WithTags(fmt.Sprintf("view:%s", name))
|
||||
return view
|
||||
}
|
||||
|
||||
|
|
@ -511,17 +511,17 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
|
|||
}
|
||||
|
||||
// Import bulk imports data.
|
||||
func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error {
|
||||
func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error {
|
||||
// Determine quantum if timestamps are set.
|
||||
q := f.TimeQuantum()
|
||||
if hasTime(timestamps) && q == "" {
|
||||
return errors.New("time quantum not set in either database or frame")
|
||||
return errors.New("time quantum not set in either index or frame")
|
||||
}
|
||||
|
||||
// Split import data by fragment.
|
||||
dataByFragment := make(map[importKey]importData)
|
||||
for i := range bitmapIDs {
|
||||
bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i]
|
||||
for i := range rowIDs {
|
||||
rowID, columnID, timestamp := rowIDs[i], columnIDs[i], timestamps[i]
|
||||
|
||||
var standard, inverse []string
|
||||
if timestamp == nil {
|
||||
|
|
@ -534,30 +534,37 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time)
|
|||
|
||||
// Attach bit to each standard view.
|
||||
for _, name := range standard {
|
||||
key := importKey{View: name, Slice: profileID / SliceWidth}
|
||||
key := importKey{View: name, Slice: columnID / SliceWidth}
|
||||
data := dataByFragment[key]
|
||||
data.BitmapIDs = append(data.BitmapIDs, bitmapID)
|
||||
data.ProfileIDs = append(data.ProfileIDs, profileID)
|
||||
data.RowIDs = append(data.RowIDs, rowID)
|
||||
data.ColumnIDs = append(data.ColumnIDs, columnID)
|
||||
dataByFragment[key] = data
|
||||
}
|
||||
|
||||
// Attach reversed bits to each inverse view.
|
||||
for _, name := range inverse {
|
||||
key := importKey{View: name, Slice: bitmapID / SliceWidth}
|
||||
data := dataByFragment[key]
|
||||
data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed
|
||||
data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed
|
||||
dataByFragment[key] = data
|
||||
if f.inverseEnabled {
|
||||
// Attach reversed bits to each inverse view.
|
||||
for _, name := range inverse {
|
||||
key := importKey{View: name, Slice: rowID / SliceWidth}
|
||||
data := dataByFragment[key]
|
||||
data.RowIDs = append(data.RowIDs, columnID) // reversed
|
||||
data.ColumnIDs = append(data.ColumnIDs, rowID) // reversed
|
||||
dataByFragment[key] = data
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Import into each fragment.
|
||||
for key, data := range dataByFragment {
|
||||
// Skip inverse data if inverse is not enabled.
|
||||
if !f.inverseEnabled && IsInverseView(key.View) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Re-sort data for inverse views.
|
||||
if IsInverseView(key.View) {
|
||||
sort.Sort(importBitSet{
|
||||
bitmapIDs: data.BitmapIDs,
|
||||
profileIDs: data.ProfileIDs,
|
||||
rowIDs: data.RowIDs,
|
||||
columnIDs: data.ColumnIDs,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
@ -571,7 +578,7 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time)
|
|||
return err
|
||||
}
|
||||
|
||||
if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil {
|
||||
if err := frag.Import(data.RowIDs, data.ColumnIDs); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
|
@ -593,9 +600,11 @@ func encodeFrame(f *Frame) *internal.Frame {
|
|||
return &internal.Frame{
|
||||
Name: f.name,
|
||||
Meta: &internal.FrameMeta{
|
||||
TimeQuantum: string(f.timeQuantum),
|
||||
RowLabel: f.rowLabel,
|
||||
CacheSize: f.cacheSize,
|
||||
RowLabel: f.rowLabel,
|
||||
InverseEnabled: f.inverseEnabled,
|
||||
CacheType: f.cacheType,
|
||||
CacheSize: f.cacheSize,
|
||||
TimeQuantum: string(f.timeQuantum),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
|
@ -627,18 +636,29 @@ type FrameOptions struct {
|
|||
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
|
||||
}
|
||||
|
||||
// Encode converts o into its internal representation.
|
||||
func (o *FrameOptions) Encode() *internal.FrameMeta {
|
||||
return &internal.FrameMeta{
|
||||
RowLabel: o.RowLabel,
|
||||
InverseEnabled: o.InverseEnabled,
|
||||
CacheType: o.CacheType,
|
||||
CacheSize: o.CacheSize,
|
||||
TimeQuantum: string(o.TimeQuantum),
|
||||
}
|
||||
}
|
||||
|
||||
// importBitSet represents slices of row and column ids.
|
||||
// This is used to sort data during import.
|
||||
type importBitSet struct {
|
||||
bitmapIDs, profileIDs []uint64
|
||||
rowIDs, columnIDs []uint64
|
||||
}
|
||||
|
||||
func (p importBitSet) Swap(i, j int) {
|
||||
p.bitmapIDs[i], p.bitmapIDs[j] = p.bitmapIDs[j], p.bitmapIDs[i]
|
||||
p.profileIDs[i], p.profileIDs[j] = p.profileIDs[j], p.profileIDs[i]
|
||||
p.rowIDs[i], p.rowIDs[j] = p.rowIDs[j], p.rowIDs[i]
|
||||
p.columnIDs[i], p.columnIDs[j] = p.columnIDs[j], p.columnIDs[i]
|
||||
}
|
||||
func (p importBitSet) Len() int { return len(p.bitmapIDs) }
|
||||
func (p importBitSet) Less(i, j int) bool { return p.bitmapIDs[i] < p.bitmapIDs[j] }
|
||||
func (p importBitSet) Len() int { return len(p.rowIDs) }
|
||||
func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] }
|
||||
|
||||
// Cache types.
|
||||
const (
|
||||
|
|
|
|||
|
|
@ -60,7 +60,7 @@ func TestFrame_NameRestriction(t *testing.T) {
|
|||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
frame, err := pilosa.NewFrame(path, "d", ".meta")
|
||||
frame, err := pilosa.NewFrame(path, "i", ".meta")
|
||||
if frame != nil {
|
||||
t.Fatalf("unexpected frame name %s", err)
|
||||
}
|
||||
|
|
@ -77,7 +77,7 @@ func NewFrame() *Frame {
|
|||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
frame, err := pilosa.NewFrame(path, "d", "f")
|
||||
frame, err := pilosa.NewFrame(path, "i", "f")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
@ -99,15 +99,15 @@ func (f *Frame) Close() error {
|
|||
return f.Frame.Close()
|
||||
}
|
||||
|
||||
// Reopen closes the database and reopens it.
|
||||
// Reopen closes the index and reopens it.
|
||||
func (f *Frame) Reopen() error {
|
||||
var err error
|
||||
if err := f.Frame.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
path, db, name := f.Path(), f.DB(), f.Name()
|
||||
f.Frame, err = pilosa.NewFrame(path, db, name)
|
||||
path, index, name := f.Path(), f.Index(), f.Name()
|
||||
f.Frame, err = pilosa.NewFrame(path, index, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -119,8 +119,8 @@ func (f *Frame) Reopen() error {
|
|||
}
|
||||
|
||||
// MustSetBit sets a bit on the frame. Panic on error.
|
||||
func (f *Frame) MustSetBit(view string, bitmapID, profileID uint64, t *time.Time) (changed bool) {
|
||||
changed, err := f.SetBit(view, bitmapID, profileID, t)
|
||||
func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) {
|
||||
changed, err := f.SetBit(view, rowID, columnID, t)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,14 +14,6 @@ import (
|
|||
"github.com/pilosa/pilosa/internal"
|
||||
)
|
||||
|
||||
// StateHandler specifies two methods which an object must implement to share
|
||||
// state in the cluster. These are used by the GossipNodeSet to implement the
|
||||
// LocalState and MergeRemoteState methods of memberlist.Delegate
|
||||
type StateHandler interface {
|
||||
LocalState() (proto.Message, error)
|
||||
HandleRemoteState(proto.Message) error
|
||||
}
|
||||
|
||||
// GossipNodeSet represents a gossip implementation of NodeSet using memberlist
|
||||
// GossipNodeSet also represents a gossip implementation of pilosa.Broadcaster
|
||||
// GossipNodeSet also represents an implementation of memberlist.Delegate
|
||||
|
|
@ -31,8 +23,8 @@ type GossipNodeSet struct {
|
|||
|
||||
broadcasts *memberlist.TransmitLimitedQueue
|
||||
|
||||
stateHandler StateHandler
|
||||
config *GossipConfig
|
||||
statusHandler pilosa.StatusHandler
|
||||
config *GossipConfig
|
||||
|
||||
// The writer for any logging.
|
||||
LogOutput io.Writer
|
||||
|
|
@ -89,7 +81,7 @@ type GossipConfig struct {
|
|||
}
|
||||
|
||||
// NewGossipNodeSet returns a new instance of GossipNodeSet.
|
||||
func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh StateHandler) *GossipNodeSet {
|
||||
func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh pilosa.StatusHandler) *GossipNodeSet {
|
||||
g := &GossipNodeSet{
|
||||
LogOutput: os.Stderr,
|
||||
}
|
||||
|
|
@ -106,7 +98,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed
|
|||
g.config.memberlistConfig.AdvertisePort = gossipPort
|
||||
g.config.memberlistConfig.Delegate = g
|
||||
|
||||
g.stateHandler = sh
|
||||
g.statusHandler = sh
|
||||
|
||||
return g
|
||||
}
|
||||
|
|
@ -154,13 +146,6 @@ func (g *GossipNodeSet) SendAsync(pb proto.Message) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (g *GossipNodeSet) Receive(pb proto.Message) error {
|
||||
if err := g.handler.ReceiveMessage(pb); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// implementation of the memberlist.Delegate interface
|
||||
func (g *GossipNodeSet) NodeMeta(limit int) []byte {
|
||||
return []byte{}
|
||||
|
|
@ -172,7 +157,7 @@ func (g *GossipNodeSet) NotifyMsg(b []byte) {
|
|||
g.logger().Printf("unmarshal message error: %s", err)
|
||||
return
|
||||
}
|
||||
if err := g.Receive(m); err != nil {
|
||||
if err := g.handler.ReceiveMessage(m); err != nil {
|
||||
g.logger().Printf("receive message error: %s", err)
|
||||
return
|
||||
}
|
||||
|
|
@ -183,7 +168,7 @@ func (g *GossipNodeSet) GetBroadcasts(overhead, limit int) [][]byte {
|
|||
}
|
||||
|
||||
func (g *GossipNodeSet) LocalState(join bool) []byte {
|
||||
pb, err := g.stateHandler.LocalState()
|
||||
pb, err := g.statusHandler.LocalStatus()
|
||||
if err != nil {
|
||||
g.logger().Printf("error getting local state, err=%s", err)
|
||||
return []byte{}
|
||||
|
|
@ -200,12 +185,12 @@ func (g *GossipNodeSet) LocalState(join bool) []byte {
|
|||
|
||||
func (g *GossipNodeSet) MergeRemoteState(buf []byte, join bool) {
|
||||
// Unmarshal nodestate data.
|
||||
var pb internal.NodeState
|
||||
var pb internal.NodeStatus
|
||||
if err := proto.Unmarshal(buf, &pb); err != nil {
|
||||
g.logger().Printf("error unmarshalling nodestate data, err=%s", err)
|
||||
return
|
||||
}
|
||||
err := g.stateHandler.HandleRemoteState(&pb)
|
||||
err := g.statusHandler.HandleRemoteStatus(&pb)
|
||||
if err != nil {
|
||||
g.logger().Printf("merge state error: %s", err)
|
||||
}
|
||||
|
|
|
|||
546
handler.go
546
handler.go
File diff suppressed because it is too large
Load diff
|
|
@ -6,21 +6,21 @@ import (
|
|||
"testing"
|
||||
)
|
||||
|
||||
// Test custom UnmarshalJSON for postDBRequest object
|
||||
func TestPostDBRequestUnmarshalJSON(t *testing.T) {
|
||||
// Test custom UnmarshalJSON for postIndexRequest object
|
||||
func TestPostIndexRequestUnmarshalJSON(t *testing.T) {
|
||||
tests := []struct {
|
||||
json string
|
||||
expected postDBRequest
|
||||
expected postIndexRequest
|
||||
err string
|
||||
}{
|
||||
{json: `{"options": {}}`, expected: postDBRequest{Options: DBOptions{}}},
|
||||
{json: `{"options": {}}`, expected: postIndexRequest{Options: IndexOptions{}}},
|
||||
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
|
||||
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
|
||||
{json: `{"options": {"columnLabel": "test"}}`, expected: postDBRequest{Options: DBOptions{ColumnLabel: "test"}}},
|
||||
{json: `{"options": {"columnLabl": "test"}}`, err: "invalid key for options {columnLabl:test}"},
|
||||
{json: `{"options": {"columnLabel": "test"}}`, expected: postIndexRequest{Options: IndexOptions{ColumnLabel: "test"}}},
|
||||
{json: `{"options": {"columnLabl": "test"}}`, err: "Unknown key: columnLabl:test"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
actual := &postDBRequest{}
|
||||
actual := &postIndexRequest{}
|
||||
err := json.Unmarshal([]byte(test.json), actual)
|
||||
|
||||
if err != nil {
|
||||
|
|
@ -51,9 +51,12 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) {
|
|||
}{
|
||||
{json: `{"options": {}}`, expected: postFrameRequest{Options: FrameOptions{}}},
|
||||
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
|
||||
{json: `{"option": {}}`, err: "Unknown key: {option:map[]}"},
|
||||
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
|
||||
{json: `{"options": {"rowLabel": "test"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test"}}},
|
||||
{json: `{"options": {"rowLabl": "test"}}`, err: "invalid key for options {rowLabl:test}"},
|
||||
{json: `{"options": {"rowLabl": "test"}}`, err: "Unknown key: rowLabl:test"},
|
||||
{json: `{"options": {"rowLabel": "test", "inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true}}},
|
||||
{json: `{"options": {"rowLabel": "test", "inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{RowLabel: "test", InverseEnabled: true, CacheType: "type"}}},
|
||||
{json: `{"options": {"rowLabel": "test", "inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
|
||||
}
|
||||
for _, test := range tests {
|
||||
actual := &postFrameRequest{}
|
||||
|
|
|
|||
428
handler_test.go
428
handler_test.go
|
|
@ -39,71 +39,71 @@ func TestHandler_NotFound(t *testing.T) {
|
|||
|
||||
// Ensure the handler can return the schema.
|
||||
func TestHandler_Schema(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
|
||||
d1 := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{})
|
||||
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
|
||||
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
|
||||
|
||||
if f, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
|
||||
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f, err := d1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
|
||||
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := d0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
|
||||
if _, err := i0.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Holder = hldr.Holder
|
||||
h.Cluster = NewCluster(1)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"dbs":[{"name":"d0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"d1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
|
||||
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
|
||||
t.Fatalf("unexpected body: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the handler can return the maxslice map.
|
||||
func TestHandler_MaxSlices(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
|
||||
idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2)
|
||||
idx.MustCreateFragmentIfNotExists("d0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4)
|
||||
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4)
|
||||
|
||||
idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1)
|
||||
idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2)
|
||||
idx.MustCreateFragmentIfNotExists("d1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
|
||||
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1)
|
||||
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2)
|
||||
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Holder = hldr.Holder
|
||||
h.Cluster = NewCluster(1)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" {
|
||||
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
|
||||
t.Fatalf("unexpected body: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the handler can return the maxslice map for the inverse views.
|
||||
func TestHandler_MaxSlices_Inverse(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
f0, err := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true})
|
||||
f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -115,7 +115,7 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f1, err := idx.MustCreateDBIfNotExists("d1", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true})
|
||||
f1, err := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -128,28 +128,28 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
|
|||
}
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Holder = hldr.Holder
|
||||
h.Cluster = NewCluster(1)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"maxSlices":{"d0":3,"d1":0}}`+"\n" {
|
||||
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
|
||||
t.Fatalf("unexpected body: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the handler can accept URL arguments.
|
||||
func TestHandler_Query_Args_URL(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Index = idx.Index
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if db != "db0" {
|
||||
t.Fatalf("unexpected db: %s", db)
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if index != "idx0" {
|
||||
t.Fatalf("unexpected index: %s", index)
|
||||
} else if query.String() != `Count(Bitmap(id=100))` {
|
||||
t.Fatalf("unexpected query: %s", query.String())
|
||||
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
|
||||
|
|
@ -159,7 +159,7 @@ func TestHandler_Query_Args_URL(t *testing.T) {
|
|||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code, w.Body.String())
|
||||
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
|
||||
|
|
@ -169,15 +169,15 @@ func TestHandler_Query_Args_URL(t *testing.T) {
|
|||
|
||||
// Ensure the handler can accept arguments via protobufs.
|
||||
func TestHandler_Query_Args_Protobuf(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Index = idx.Index
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if db != "db0" {
|
||||
t.Fatalf("unexpected db: %s", db)
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
if index != "idx0" {
|
||||
t.Fatalf("unexpected index: %s", index)
|
||||
} else if query.String() != `Count(Bitmap(id=100))` {
|
||||
t.Fatalf("unexpected query: %s", query.String())
|
||||
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
|
||||
|
|
@ -196,7 +196,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
|
|||
}
|
||||
|
||||
// Generate protobuf request.
|
||||
req := MustNewHTTPRequest("POST", "/db/db0/query", bytes.NewReader(reqBody))
|
||||
req := MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody))
|
||||
req.Header.Set("Content-Type", "application/x-protobuf")
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
|
|
@ -209,14 +209,14 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
|
|||
// Ensure the handler returns an error when parsing bad arguments.
|
||||
func TestHandler_Query_Args_Err(t *testing.T) {
|
||||
w := httptest.NewRecorder()
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Index = idx.Index
|
||||
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
|
||||
h.Holder = hldr.Holder
|
||||
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
|
||||
|
|
@ -226,18 +226,18 @@ func TestHandler_Query_Args_Err(t *testing.T) {
|
|||
|
||||
// Ensure the handler can execute a query with a uint64 response as JSON.
|
||||
func TestHandler_Query_Uint64_JSON(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
return []interface{}{uint64(100)}, nil
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
|
||||
|
|
@ -247,18 +247,18 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
|
|||
|
||||
// Ensure the handler can execute a query with a uint64 response as protobufs.
|
||||
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
return []interface{}{uint64(100)}, nil
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader("Count(Bitmap(id=100))"))
|
||||
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))"))
|
||||
r.Header.Set("Accept", "application/x-protobuf")
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusOK {
|
||||
|
|
@ -275,20 +275,20 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
|
|||
|
||||
// Ensure the handler can execute a query that returns a bitmap as JSON.
|
||||
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
|
||||
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
|
||||
return []interface{}{bm}, nil
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader("Bitmap(id=100)")))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" {
|
||||
|
|
@ -296,55 +296,55 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure the handler can execute a query that returns a bitmap with profiles as JSON.
|
||||
func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
|
||||
idx := NewIndex()
|
||||
defer idx.Close()
|
||||
// Ensure the handler can execute a query that returns a bitmap with column attributes as JSON.
|
||||
func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
|
||||
hldr := NewHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Create database and set profile attributes.
|
||||
db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
|
||||
// Create index and set column attributes.
|
||||
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := db.ProfileAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
|
||||
} else if err := index.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := db.ProfileAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil {
|
||||
} else if err := index.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Holder = hldr.Holder
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
|
||||
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
|
||||
return []interface{}{bm}, nil
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query?profiles=true", strings.NewReader("Bitmap(id=100)")))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
|
||||
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
|
||||
t.Fatalf("unexpected body: %s", body)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the handler can execute a query that returns a bitmap as protobuf.
|
||||
func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
|
||||
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
|
||||
return []interface{}{bm}, nil
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader("Bitmap(id=100)"))
|
||||
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))
|
||||
r.Header.Set("Accept", "application/x-protobuf")
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusOK {
|
||||
|
|
@ -367,23 +367,23 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure the handler can execute a query that returns a bitmap with profiles as protobuf.
|
||||
func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
|
||||
idx := NewIndex()
|
||||
defer idx.Close()
|
||||
// Ensure the handler can execute a query that returns a bitmap with column attributes as protobuf.
|
||||
func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
|
||||
hldr := NewHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Create database and set profile attributes.
|
||||
db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
|
||||
// Create index and set column attributes.
|
||||
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
|
||||
} else if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Holder = hldr.Holder
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
|
||||
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
|
||||
return []interface{}{bm}, nil
|
||||
|
|
@ -391,15 +391,15 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
|
|||
|
||||
// Encode request body.
|
||||
buf, err := proto.Marshal(&internal.QueryRequest{
|
||||
Query: "Bitmap(id=100)",
|
||||
Profiles: true,
|
||||
Query: "Bitmap(id=100)",
|
||||
ColumnAttrs: true,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r := MustNewHTTPRequest("POST", "/db/d/query", bytes.NewReader(buf))
|
||||
r := MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf))
|
||||
r.Header.Set("Content-Type", "application/x-protobuf")
|
||||
r.Header.Set("Accept", "application/x-protobuf")
|
||||
h.ServeHTTP(w, r)
|
||||
|
|
@ -423,12 +423,12 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
|
|||
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
|
||||
}
|
||||
|
||||
if a := resp.Profiles; len(a) != 1 {
|
||||
t.Fatalf("unexpected profiles length: %d", len(a))
|
||||
if a := resp.ColumnAttrSets; len(a) != 1 {
|
||||
t.Fatalf("unexpected column attributes length: %d", len(a))
|
||||
} else if a[0].ID != 1 {
|
||||
t.Fatalf("unexpected id: %d", a[0].ID)
|
||||
} else if len(a[0].Attrs) != 1 {
|
||||
t.Fatalf("unexpected profile attr length: %d", len(a))
|
||||
t.Fatalf("unexpected column attr length: %d", len(a))
|
||||
} else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" {
|
||||
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
|
||||
}
|
||||
|
|
@ -436,13 +436,13 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
|
|||
|
||||
// Ensure the handler can execute a query that returns pairs as JSON.
|
||||
func TestHandler_Query_Pairs_JSON(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
return []interface{}{[]pilosa.Pair{
|
||||
{ID: 1, Count: 2},
|
||||
{ID: 3, Count: 4},
|
||||
|
|
@ -450,7 +450,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
|
|||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`TopN(frame=x, n=2)`)))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" {
|
||||
|
|
@ -459,14 +459,14 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
|
|||
}
|
||||
|
||||
// Ensure the handler can execute a query that returns pairs as protobuf.
|
||||
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
return []interface{}{[]pilosa.Pair{
|
||||
{ID: 1, Count: 2},
|
||||
{ID: 3, Count: 4},
|
||||
|
|
@ -474,7 +474,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
|
|||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`TopN(frame=x, n=2)`))
|
||||
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
|
||||
r.Header.Set("Accept", "application/x-protobuf")
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusOK {
|
||||
|
|
@ -491,18 +491,18 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
|
|||
|
||||
// Ensure the handler can return an error as JSON.
|
||||
func TestHandler_Query_Err_JSON(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
return nil, errors.New("marker")
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`Bitmap(id=100)`)))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`)))
|
||||
if w.Code != http.StatusInternalServerError {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" {
|
||||
|
|
@ -512,18 +512,18 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
|
|||
|
||||
// Ensure the handler can return an error as protobuf.
|
||||
func TestHandler_Query_Err_Protobuf(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
h.Holder = hldr.Holder
|
||||
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
return nil, errors.New("marker")
|
||||
}
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r := MustNewHTTPRequest("POST", "/db/d/query", strings.NewReader(`TopN(frame=x, n=2)`))
|
||||
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
|
||||
r.Header.Set("Accept", "application/x-protobuf")
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusInternalServerError {
|
||||
|
|
@ -540,14 +540,14 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) {
|
|||
|
||||
// Ensure the handler returns "method not allowed" for non-POST queries.
|
||||
func TestHandler_Query_MethodNotAllowed(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
h.Holder = hldr.Holder
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/db/d/query", nil))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i/query", nil))
|
||||
if w.Code != http.StatusMethodNotAllowed {
|
||||
t.Fatalf("invalid status: %d", w.Code)
|
||||
}
|
||||
|
|
@ -562,7 +562,7 @@ func TestHandler_Query_ErrParse(t *testing.T) {
|
|||
h.Index = idx.Index
|
||||
h.Cluster = NewCluster(1)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/db0/query?slices=0,1", strings.NewReader("bad_fn(")))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
|
||||
if w.Code != http.StatusBadRequest {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{"error":"expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" {
|
||||
|
|
@ -570,22 +570,22 @@ func TestHandler_Query_ErrParse(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure the handler can delete a database.
|
||||
func TestHandler_DB_Delete(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
// Ensure the handler can delete an index.
|
||||
func TestHandler_Index_Delete(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
s := NewServer()
|
||||
s.Handler.Index = idx.Index
|
||||
s.Handler.Holder = hldr.Holder
|
||||
defer s.Close()
|
||||
|
||||
// Create database.
|
||||
if _, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}); err != nil {
|
||||
// Create index.
|
||||
if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Send request to delete database.
|
||||
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/db/d", strings.NewReader("")))
|
||||
// Send request to delete index.
|
||||
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -600,103 +600,103 @@ func TestHandler_DB_Delete(t *testing.T) {
|
|||
t.Fatalf("unexpected response body: %s", buf)
|
||||
}
|
||||
|
||||
// Verify database is gone.
|
||||
if idx.DB("d") != nil {
|
||||
t.Fatal("expected nil database")
|
||||
// Verify index is gone.
|
||||
if hldr.Index("i") != nil {
|
||||
t.Fatal("expected nil index")
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure handler can delete a frame.
|
||||
func TestHandler_DeleteFrame(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
d0 := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
|
||||
if _, err := d0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
|
||||
if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Holder = hldr.Holder
|
||||
h.Cluster = NewCluster(1)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/db/d0/frame/f1", strings.NewReader("")))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader("")))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{}`+"\n" {
|
||||
t.Fatalf("unexpected body: %s", body)
|
||||
} else if f := idx.DB("d0").Frame("f1"); f != nil {
|
||||
} else if f := hldr.Index("i0").Frame("f1"); f != nil {
|
||||
t.Fatal("expected nil frame")
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure handler can set the DB time quantum.
|
||||
func TestHandler_SetDBTimeQuantum(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{})
|
||||
// Ensure handler can set the Index time quantum.
|
||||
func TestHandler_SetIndexTimeQuantum(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Holder = hldr.Holder
|
||||
h.Cluster = NewCluster(1)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{}`+"\n" {
|
||||
t.Fatalf("unexpected body: %s", body)
|
||||
} else if q := idx.DB("d0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
|
||||
} else if q := hldr.Index("i0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
|
||||
t.Fatalf("unexpected time quantum: %s", q)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure handler can set the frame time quantum.
|
||||
func TestHandler_SetFrameTimeQuantum(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Create frame.
|
||||
if _, err := idx.MustCreateDBIfNotExists("d0", pilosa.DBOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil {
|
||||
if _, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
h := NewHandler()
|
||||
h.Index = idx.Index
|
||||
h.Holder = hldr.Holder
|
||||
h.Cluster = NewCluster(1)
|
||||
w := httptest.NewRecorder()
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/d0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
|
||||
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
} else if body := w.Body.String(); body != `{}`+"\n" {
|
||||
t.Fatalf("unexpected body: %s", body)
|
||||
} else if q := idx.DB("d0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
|
||||
} else if q := hldr.Index("i0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
|
||||
t.Fatalf("unexpected time quantum: %s", q)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure the handler can return data in differing blocks for a database.
|
||||
func TestHandler_DB_AttrStore_Diff(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
// Ensure the handler can return data in differing blocks for an index.
|
||||
func TestHandler_Index_AttrStore_Diff(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
s := NewServer()
|
||||
s.Handler.Index = idx.Index
|
||||
s.Handler.Holder = hldr.Holder
|
||||
defer s.Close()
|
||||
|
||||
// Set attributes on the database.
|
||||
db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
|
||||
// Set attributes on the index.
|
||||
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
|
||||
if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := db.ProfileAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
|
||||
} else if err := index.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := db.ProfileAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
|
||||
} else if err := index.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Retrieve block checksums.
|
||||
blks, err := db.ProfileAttrStore().Blocks()
|
||||
blks, err := index.ColumnAttrStore().Blocks()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -707,7 +707,7 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) {
|
|||
|
||||
// Send block checksums to determine diff.
|
||||
resp, err := http.Post(
|
||||
s.URL+"/db/d/attr/diff",
|
||||
s.URL+"/index/i/attr/diff",
|
||||
"application/json",
|
||||
strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`),
|
||||
)
|
||||
|
|
@ -724,29 +724,29 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) {
|
|||
|
||||
// Ensure the handler can return data in differing blocks for a frame.
|
||||
func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
s := NewServer()
|
||||
s.Handler.Index = idx.Index
|
||||
s.Handler.Holder = hldr.Holder
|
||||
defer s.Close()
|
||||
|
||||
// Set attributes on the database.
|
||||
d := idx.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
|
||||
f, err := d.CreateFrameIfNotExists("meta", pilosa.FrameOptions{})
|
||||
// Set attributes on the index.
|
||||
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
|
||||
f, err := idx.CreateFrameIfNotExists("meta", pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := f.BitmapAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
|
||||
if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := f.BitmapAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
|
||||
} else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := f.BitmapAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
|
||||
} else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Retrieve block checksums.
|
||||
blks, err := f.BitmapAttrStore().Blocks()
|
||||
blks, err := f.RowAttrStore().Blocks()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -757,7 +757,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
|
|||
|
||||
// Send block checksums to determine diff.
|
||||
resp, err := http.Post(
|
||||
s.URL+"/db/d/frame/meta/attr/diff",
|
||||
s.URL+"/index/i/frame/meta/attr/diff",
|
||||
"application/json",
|
||||
strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`),
|
||||
)
|
||||
|
|
@ -774,19 +774,19 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
|
|||
|
||||
// Ensure the handler can backup a fragment and then restore it.
|
||||
func TestHandler_Fragment_BackupRestore(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
s := NewServer()
|
||||
s.Handler.Index = idx.Index
|
||||
s.Handler.Holder = hldr.Holder
|
||||
defer s.Close()
|
||||
|
||||
// Set bits in the index.
|
||||
f0 := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
|
||||
f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
|
||||
f0.MustSetBits(100, 1, 2, 3)
|
||||
|
||||
// Begin backing up from slice d/f/0.
|
||||
resp, err := http.Get(s.URL + "/fragment/data?db=d&frame=f&view=standard&slice=0")
|
||||
// Begin backing up from slice i/f/0.
|
||||
resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -798,12 +798,12 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
|
|||
}
|
||||
|
||||
// Create frame.
|
||||
if _, err := idx.MustCreateDBIfNotExists("x", pilosa.DBOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil {
|
||||
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Restore backup to slice x/y/0.
|
||||
if resp, err := http.Post(s.URL+"/fragment/data?db=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil {
|
||||
if resp, err := http.Post(s.URL+"/fragment/data?index=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if resp.StatusCode != http.StatusOK {
|
||||
resp.Body.Close()
|
||||
|
|
@ -813,10 +813,10 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
|
|||
}
|
||||
|
||||
// Verify data is correctly restored.
|
||||
f1 := idx.Fragment("x", "y", pilosa.ViewStandard, 0)
|
||||
f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0)
|
||||
if f1 == nil {
|
||||
t.Fatal("fragment x/y/standard/0 not created")
|
||||
} else if bits := f1.Bitmap(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) {
|
||||
} else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) {
|
||||
t.Fatalf("unexpected restored bits: %+v", bits)
|
||||
}
|
||||
}
|
||||
|
|
@ -851,7 +851,7 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
|
|||
h.Cluster.ReplicaN = 2
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
r := MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
|
||||
r := MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil)
|
||||
h.ServeHTTP(w, r)
|
||||
if w.Code != http.StatusOK {
|
||||
t.Fatalf("unexpected status code: %d", w.Code)
|
||||
|
|
@ -899,13 +899,13 @@ func NewHandler() *Handler {
|
|||
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
|
||||
type HandlerExecutor struct {
|
||||
cluster *pilosa.Cluster
|
||||
ExecuteFn func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
|
||||
ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
|
||||
}
|
||||
|
||||
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
|
||||
|
||||
func (c *HandlerExecutor) Execute(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
return c.ExecuteFn(ctx, db, query, slices, opt)
|
||||
func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
return c.ExecuteFn(ctx, index, query, slices, opt)
|
||||
}
|
||||
|
||||
// Server represents a test wrapper for httptest.Server.
|
||||
|
|
@ -971,57 +971,3 @@ func MustReadAll(r io.Reader) []byte {
|
|||
}
|
||||
return buf
|
||||
}
|
||||
|
||||
/*
|
||||
// TODO: move this test to messenger.go (with NewServer())
|
||||
|
||||
// Ensure that an HTTP message sent to the cluster reaches all nodes.
|
||||
func TestHTTPNodeSet_Base(t *testing.T) {
|
||||
|
||||
// servers
|
||||
s1 := NewServer()
|
||||
s1.Messenger = pilosa.NewMessenger()
|
||||
n1 := NewHTTPMessageBroker()
|
||||
n1.messenger = s1.Messenger
|
||||
s1.Messenger.Broker = n1
|
||||
|
||||
s2 := NewServer()
|
||||
s2.Messenger = pilosa.NewMessenger()
|
||||
n2 := NewHTTPMessageBroker()
|
||||
n2.messenger = s2.Messenger
|
||||
s2.Messenger.Broker = n2
|
||||
|
||||
s3 := NewServer()
|
||||
s3.Messenger = pilosa.NewMessenger()
|
||||
n3 := NewHTTPMessageBroker()
|
||||
n3.messenger = s3.Messenger
|
||||
s3.Messenger.Broker = n3
|
||||
|
||||
nodes := []*pilosa.Node{
|
||||
{Host: s1.Host()},
|
||||
{Host: s2.Host()},
|
||||
{Host: s3.Host()},
|
||||
}
|
||||
|
||||
// message
|
||||
msg := &internal.CreateSliceMessage{
|
||||
DB: "d",
|
||||
Slice: 8,
|
||||
}
|
||||
|
||||
// send message
|
||||
if err := s1.Messenger.SendMessage(msg, ""); err != nil {
|
||||
t.Fatalf("failure sending message: %s", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(mb1.messageReceived, msg) {
|
||||
t.Fatalf("unexpected message received by node1: %s", mb1.messageReceived)
|
||||
}
|
||||
if !reflect.DeepEqual(mb2.messageReceived, msg) {
|
||||
t.Fatalf("unexpected message received by node2: %s", mb2.messageReceived)
|
||||
}
|
||||
if !reflect.DeepEqual(mb3.messageReceived, msg) {
|
||||
t.Fatalf("unexpected message received by node3: %s", mb3.messageReceived)
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
|
|
|||
547
holder.go
Normal file
547
holder.go
Normal file
|
|
@ -0,0 +1,547 @@
|
|||
package pilosa
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
// DefaultCacheFlushInterval is the default value for Fragment.CacheFlushInterval.
|
||||
const DefaultCacheFlushInterval = 1 * time.Minute
|
||||
|
||||
// Holder represents a container for indexes.
|
||||
type Holder struct {
|
||||
mu sync.Mutex
|
||||
|
||||
// Indexes by name.
|
||||
indexes map[string]*Index
|
||||
|
||||
Broadcaster Broadcaster
|
||||
// Close management
|
||||
wg sync.WaitGroup
|
||||
closing chan struct{}
|
||||
|
||||
// Stats
|
||||
Stats StatsClient
|
||||
|
||||
// Data directory path.
|
||||
Path string
|
||||
|
||||
// The interval at which the cached row ids are persisted to disk.
|
||||
CacheFlushInterval time.Duration
|
||||
|
||||
LogOutput io.Writer
|
||||
}
|
||||
|
||||
// NewHolder returns a new instance of Holder.
|
||||
func NewHolder() *Holder {
|
||||
return &Holder{
|
||||
indexes: make(map[string]*Index),
|
||||
closing: make(chan struct{}, 0),
|
||||
|
||||
Stats: NopStatsClient,
|
||||
|
||||
CacheFlushInterval: DefaultCacheFlushInterval,
|
||||
|
||||
LogOutput: os.Stderr,
|
||||
}
|
||||
}
|
||||
|
||||
// Open initializes the root data directory for the holder.
|
||||
func (h *Holder) Open() error {
|
||||
if err := os.MkdirAll(h.Path, 0777); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Open path to read all index directories.
|
||||
f, err := os.Open(h.Path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
fis, err := f.Readdir(0)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, fi := range fis {
|
||||
if !fi.IsDir() {
|
||||
continue
|
||||
}
|
||||
|
||||
h.logger().Printf("opening index: %s", filepath.Base(fi.Name()))
|
||||
|
||||
index, err := h.newIndex(h.IndexPath(filepath.Base(fi.Name())), filepath.Base(fi.Name()))
|
||||
if err == ErrName {
|
||||
h.logger().Printf("ERROR opening index: %s, err=%s", fi.Name(), err)
|
||||
continue
|
||||
} else if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := index.Open(); err != nil {
|
||||
if err == ErrName {
|
||||
h.logger().Printf("ERROR opening index: %s, err=%s", index.Name(), err)
|
||||
continue
|
||||
}
|
||||
return fmt.Errorf("open index: name=%s, err=%s", index.Name(), err)
|
||||
}
|
||||
h.indexes[index.Name()] = index
|
||||
|
||||
h.Stats.Count("indexN", 1)
|
||||
}
|
||||
|
||||
// Periodically flush cache.
|
||||
h.wg.Add(1)
|
||||
go func() { defer h.wg.Done(); h.monitorCacheFlush() }()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close closes all open fragments.
|
||||
func (h *Holder) Close() error {
|
||||
// Notify goroutines of closing and wait for completion.
|
||||
close(h.closing)
|
||||
h.wg.Wait()
|
||||
|
||||
for _, index := range h.indexes {
|
||||
index.Close()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MaxSlices returns MaxSlice map for all indexes.
|
||||
func (h *Holder) MaxSlices() map[string]uint64 {
|
||||
a := make(map[string]uint64)
|
||||
for _, index := range h.Indexes() {
|
||||
a[index.Name()] = index.MaxSlice()
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// MaxInverseSlices returns MaxInverseSlice map for all indexes.
|
||||
func (h *Holder) MaxInverseSlices() map[string]uint64 {
|
||||
a := make(map[string]uint64)
|
||||
for _, index := range h.Indexes() {
|
||||
a[index.Name()] = index.MaxInverseSlice()
|
||||
}
|
||||
return a
|
||||
}
|
||||
|
||||
// Schema returns schema data for all indexes and frames.
|
||||
func (h *Holder) Schema() []*IndexInfo {
|
||||
var a []*IndexInfo
|
||||
for _, index := range h.Indexes() {
|
||||
di := &IndexInfo{Name: index.Name()}
|
||||
for _, frame := range index.Frames() {
|
||||
fi := &FrameInfo{Name: frame.Name()}
|
||||
for _, view := range frame.Views() {
|
||||
fi.Views = append(fi.Views, &ViewInfo{Name: view.Name()})
|
||||
}
|
||||
sort.Sort(viewInfoSlice(fi.Views))
|
||||
di.Frames = append(di.Frames, fi)
|
||||
}
|
||||
sort.Sort(frameInfoSlice(di.Frames))
|
||||
a = append(a, di)
|
||||
}
|
||||
sort.Sort(indexInfoSlice(a))
|
||||
return a
|
||||
}
|
||||
|
||||
// IndexPath returns the path where a given index is stored.
|
||||
func (h *Holder) IndexPath(name string) string { return filepath.Join(h.Path, name) }
|
||||
|
||||
// Index returns the index by name.
|
||||
func (h *Holder) Index(name string) *Index {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.index(name)
|
||||
}
|
||||
|
||||
func (h *Holder) index(name string) *Index { return h.indexes[name] }
|
||||
|
||||
// Indexes returns a list of all indexes in the holder.
|
||||
func (h *Holder) Indexes() []*Index {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
a := make([]*Index, 0, len(h.indexes))
|
||||
for _, index := range h.indexes {
|
||||
a = append(a, index)
|
||||
}
|
||||
sort.Sort(indexSlice(a))
|
||||
|
||||
return a
|
||||
}
|
||||
|
||||
// CreateIndex creates an index.
|
||||
// An error is returned if the index already exists.
|
||||
func (h *Holder) CreateIndex(name string, opt IndexOptions) (*Index, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
// Ensure index doesn't already exist.
|
||||
if h.indexes[name] != nil {
|
||||
return nil, ErrIndexExists
|
||||
}
|
||||
return h.createIndex(name, opt)
|
||||
}
|
||||
|
||||
// CreateIndexIfNotExists returns an index by name.
|
||||
// The index is created if it does not already exist.
|
||||
func (h *Holder) CreateIndexIfNotExists(name string, opt IndexOptions) (*Index, error) {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
// Find index in cache first.
|
||||
if index := h.indexes[name]; index != nil {
|
||||
return index, nil
|
||||
}
|
||||
|
||||
return h.createIndex(name, opt)
|
||||
}
|
||||
|
||||
func (h *Holder) createIndex(name string, opt IndexOptions) (*Index, error) {
|
||||
if name == "" {
|
||||
return nil, errors.New("index name required")
|
||||
}
|
||||
|
||||
// Return index if it exists.
|
||||
if index := h.index(name); index != nil {
|
||||
return index, nil
|
||||
}
|
||||
|
||||
// Otherwise create a new index.
|
||||
index, err := h.newIndex(h.IndexPath(name), name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if err := index.Open(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Update options.
|
||||
index.SetColumnLabel(opt.ColumnLabel)
|
||||
index.SetTimeQuantum(opt.TimeQuantum)
|
||||
|
||||
h.indexes[index.Name()] = index
|
||||
|
||||
h.Stats.Count("indexN", 1)
|
||||
|
||||
return index, nil
|
||||
}
|
||||
|
||||
func (h *Holder) newIndex(path, name string) (*Index, error) {
|
||||
index, err := NewIndex(path, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
index.LogOutput = h.LogOutput
|
||||
index.Stats = h.Stats.WithTags(fmt.Sprintf("index:%s", index.Name()))
|
||||
index.broadcaster = h.Broadcaster
|
||||
return index, nil
|
||||
}
|
||||
|
||||
// DeleteIndex removes an index from the holder.
|
||||
func (h *Holder) DeleteIndex(name string) error {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
|
||||
// Ignore if index doesn't exist.
|
||||
index := h.index(name)
|
||||
if index == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close index.
|
||||
if err := index.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Delete index directory.
|
||||
if err := os.RemoveAll(h.IndexPath(name)); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Remove reference.
|
||||
delete(h.indexes, name)
|
||||
|
||||
h.Stats.Count("indexN", -1)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Frame returns the frame for an index and name.
|
||||
func (h *Holder) Frame(index, name string) *Frame {
|
||||
idx := h.Index(index)
|
||||
if idx == nil {
|
||||
return nil
|
||||
}
|
||||
return idx.Frame(name)
|
||||
}
|
||||
|
||||
// View returns the view for an index, frame, and name.
|
||||
func (h *Holder) View(index, frame, name string) *View {
|
||||
f := h.Frame(index, frame)
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
return f.View(name)
|
||||
}
|
||||
|
||||
// Fragment returns the fragment for an index, frame & slice.
|
||||
func (h *Holder) Fragment(index, frame, view string, slice uint64) *Fragment {
|
||||
v := h.View(index, frame, view)
|
||||
if v == nil {
|
||||
return nil
|
||||
}
|
||||
return v.Fragment(slice)
|
||||
}
|
||||
|
||||
// monitorCacheFlush periodically flushes all fragment caches sequentially.
|
||||
// This is run in a goroutine.
|
||||
func (h *Holder) monitorCacheFlush() {
|
||||
ticker := time.NewTicker(h.CacheFlushInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-h.closing:
|
||||
return
|
||||
case <-ticker.C:
|
||||
h.flushCaches()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) flushCaches() {
|
||||
for _, index := range h.Indexes() {
|
||||
for _, frame := range index.Frames() {
|
||||
for _, view := range frame.Views() {
|
||||
for _, fragment := range view.Fragments() {
|
||||
select {
|
||||
case <-h.closing:
|
||||
return
|
||||
default:
|
||||
}
|
||||
|
||||
if err := fragment.FlushCache(); err != nil {
|
||||
h.logger().Printf("error flushing cache: err=%s, path=%s", err, fragment.CachePath())
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Holder) logger() *log.Logger { return log.New(h.LogOutput, "", log.LstdFlags) }
|
||||
|
||||
// HolderSyncer is an active anti-entropy tool that compares the local holder
|
||||
// with a remote holder based on block checksums and resolves differences.
|
||||
type HolderSyncer struct {
|
||||
Holder *Holder
|
||||
|
||||
Host string
|
||||
Cluster *Cluster
|
||||
|
||||
// Signals that the sync should stop.
|
||||
Closing <-chan struct{}
|
||||
}
|
||||
|
||||
// Returns true if the syncer has been marked to close.
|
||||
func (s *HolderSyncer) IsClosing() bool {
|
||||
select {
|
||||
case <-s.Closing:
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
// SyncHolder compares the holder on host with the local holder and resolves differences.
|
||||
func (s *HolderSyncer) SyncHolder() error {
|
||||
// Iterate over schema in sorted order.
|
||||
for _, di := range s.Holder.Schema() {
|
||||
// Verify syncer has not closed.
|
||||
if s.IsClosing() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sync index column attributes.
|
||||
if err := s.syncIndex(di.Name); err != nil {
|
||||
return fmt.Errorf("index sync error: index=%s, err=%s", di.Name, err)
|
||||
}
|
||||
|
||||
for _, fi := range di.Frames {
|
||||
// Verify syncer has not closed.
|
||||
if s.IsClosing() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sync frame row attributes.
|
||||
if err := s.syncFrame(di.Name, fi.Name); err != nil {
|
||||
return fmt.Errorf("frame sync error: index=%s, frame=%s, err=%s", di.Name, fi.Name, err)
|
||||
}
|
||||
|
||||
for _, vi := range fi.Views {
|
||||
// Verify syncer has not closed.
|
||||
if s.IsClosing() {
|
||||
return nil
|
||||
}
|
||||
|
||||
for slice := uint64(0); slice <= s.Holder.Index(di.Name).MaxSlice(); slice++ {
|
||||
// Ignore slices that this host doesn't own.
|
||||
if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Verify syncer has not closed.
|
||||
if s.IsClosing() {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sync fragment if own it.
|
||||
if err := s.syncFragment(di.Name, fi.Name, vi.Name, slice); err != nil {
|
||||
return fmt.Errorf("fragment sync error: index=%s, frame=%s, slice=%d, err=%s", di.Name, fi.Name, slice, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncIndex synchronizes index attributes with the rest of the cluster.
|
||||
func (s *HolderSyncer) syncIndex(index string) error {
|
||||
// Retrieve index reference.
|
||||
idx := s.Holder.Index(index)
|
||||
if idx == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read block checksums.
|
||||
blks, err := idx.ColumnAttrStore().Blocks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sync with every other host.
|
||||
for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.Host) {
|
||||
client, err := NewClient(node.Host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Retrieve attributes from differing blocks.
|
||||
// Skip update and recomputation if no attributes have changed.
|
||||
m, err := client.ColumnAttrDiff(context.Background(), index, blks)
|
||||
if err != nil {
|
||||
return err
|
||||
} else if len(m) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Update local copy.
|
||||
if err := idx.ColumnAttrStore().SetBulkAttrs(m); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Recompute blocks.
|
||||
blks, err = idx.ColumnAttrStore().Blocks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncFrame synchronizes frame attributes with the rest of the cluster.
|
||||
func (s *HolderSyncer) syncFrame(index, name string) error {
|
||||
// Retrieve index reference.
|
||||
f := s.Holder.Frame(index, name)
|
||||
if f == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Read block checksums.
|
||||
blks, err := f.RowAttrStore().Blocks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sync with every other host.
|
||||
for _, node := range Nodes(s.Cluster.Nodes).FilterHost(s.Host) {
|
||||
client, err := NewClient(node.Host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Retrieve attributes from differing blocks.
|
||||
// Skip update and recomputation if no attributes have changed.
|
||||
m, err := client.RowAttrDiff(context.Background(), index, name, blks)
|
||||
if err == ErrFrameNotFound {
|
||||
continue // frame not created remotely yet, skip
|
||||
} else if err != nil {
|
||||
return err
|
||||
} else if len(m) == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
// Update local copy.
|
||||
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Recompute blocks.
|
||||
blks, err = f.RowAttrStore().Blocks()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// syncFragment synchronizes a fragment with the rest of the cluster.
|
||||
func (s *HolderSyncer) syncFragment(index, frame, view string, slice uint64) error {
|
||||
// Retrieve local frame.
|
||||
f := s.Holder.Frame(index, frame)
|
||||
if f == nil {
|
||||
return ErrFrameNotFound
|
||||
}
|
||||
|
||||
// Ensure view exists locally.
|
||||
v, err := f.CreateViewIfNotExists(view)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Ensure fragment exists locally.
|
||||
frag, err := v.CreateFragmentIfNotExists(slice)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Sync fragments together.
|
||||
fs := FragmentSyncer{
|
||||
Fragment: frag,
|
||||
Host: s.Host,
|
||||
Cluster: s.Cluster,
|
||||
Closing: s.Closing,
|
||||
}
|
||||
if err := fs.SyncFragment(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
234
holder_test.go
Normal file
234
holder_test.go
Normal file
|
|
@ -0,0 +1,234 @@
|
|||
package pilosa_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/pilosa/pilosa/pql"
|
||||
)
|
||||
|
||||
// Ensure holder can delete an index and its underlying files.
|
||||
func TestHolder_DeleteIndex(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
// Write bits to separate indexes.
|
||||
f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0)
|
||||
if _, err := f0.SetBit(100, 200); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f1 := hldr.MustCreateFragmentIfNotExists("i1", "f", pilosa.ViewStandard, 0)
|
||||
if _, err := f1.SetBit(100, 200); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Ensure i0 exists.
|
||||
if _, err := os.Stat(hldr.IndexPath("i0")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Delete i0.
|
||||
if err := hldr.DeleteIndex("i0"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Ensure i0 files are removed & i1 still exists.
|
||||
if _, err := os.Stat(hldr.IndexPath("i0")); !os.IsNotExist(err) {
|
||||
t.Fatal("expected i0 file deletion")
|
||||
} else if _, err := os.Stat(hldr.IndexPath("i1")); err != nil {
|
||||
t.Fatal("expected i1 files to still exist", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure holder can sync with a remote holder.
|
||||
func TestHolderSyncer_SyncHolder(t *testing.T) {
|
||||
cluster := NewCluster(2)
|
||||
|
||||
// Create a local holder.
|
||||
hldr0 := MustOpenHolder()
|
||||
defer hldr0.Close()
|
||||
|
||||
// Create a remote holder wrapped by an HTTP
|
||||
hldr1 := MustOpenHolder()
|
||||
defer hldr1.Close()
|
||||
s := NewServer()
|
||||
defer s.Close()
|
||||
s.Handler.Holder = hldr1.Holder
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
e := pilosa.NewExecutor()
|
||||
e.Holder = hldr1.Holder
|
||||
e.Host = cluster.Nodes[1].Host
|
||||
e.Cluster = cluster
|
||||
return e.Execute(ctx, index, query, slices, opt)
|
||||
}
|
||||
|
||||
// Mock 2-node, fully replicated cluster.
|
||||
cluster.ReplicaN = 2
|
||||
cluster.Nodes[0].Host = "localhost:0"
|
||||
cluster.Nodes[1].Host = MustParseURLHost(s.URL)
|
||||
|
||||
// Create frames on nodes.
|
||||
for _, hldr := range []*Holder{hldr0, hldr1} {
|
||||
hldr.MustCreateFrameIfNotExists("i", "f")
|
||||
hldr.MustCreateFrameIfNotExists("i", "f0")
|
||||
hldr.MustCreateFrameIfNotExists("y", "z")
|
||||
}
|
||||
|
||||
// Set data on the local holder.
|
||||
f := hldr0.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
|
||||
if _, err := f.SetBit(0, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(2, 20); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(120, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(200, 4); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f = hldr0.MustCreateFragmentIfNotExists("i", "f0", pilosa.ViewStandard, 1)
|
||||
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
hldr0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0)
|
||||
|
||||
// Set data on the remote holder.
|
||||
f = hldr1.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
|
||||
if _, err := f.SetBit(0, 4000); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(3, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(120, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f = hldr1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3)
|
||||
if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set highest slice.
|
||||
hldr0.Index("i").SetRemoteMaxSlice(1)
|
||||
hldr0.Index("y").SetRemoteMaxSlice(3)
|
||||
|
||||
// Set up syncer.
|
||||
syncer := pilosa.HolderSyncer{
|
||||
Holder: hldr0.Holder,
|
||||
Host: cluster.Nodes[0].Host,
|
||||
Cluster: cluster,
|
||||
}
|
||||
|
||||
if err := syncer.SyncHolder(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data is the same on both nodes.
|
||||
for i, hldr := range []*Holder{hldr0, hldr1} {
|
||||
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
|
||||
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
|
||||
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
|
||||
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
|
||||
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
|
||||
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
|
||||
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
|
||||
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
|
||||
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
|
||||
}
|
||||
|
||||
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
|
||||
a := f.Row(9).Bits()
|
||||
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
|
||||
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
|
||||
}
|
||||
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
|
||||
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
|
||||
}
|
||||
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3)
|
||||
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
|
||||
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Holder is a test wrapper for pilosa.Holder.
|
||||
type Holder struct {
|
||||
*pilosa.Holder
|
||||
LogOutput bytes.Buffer
|
||||
}
|
||||
|
||||
// NewHolder returns a new instance of Holder with a temporary path.
|
||||
func NewHolder() *Holder {
|
||||
path, err := ioutil.TempDir("", "pilosa-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
h := &Holder{Holder: pilosa.NewHolder()}
|
||||
h.Path = path
|
||||
h.Holder.LogOutput = &h.LogOutput
|
||||
|
||||
return h
|
||||
}
|
||||
|
||||
// MustOpenHolder creates and opens a holder at a temporary path. Panic on error.
|
||||
func MustOpenHolder() *Holder {
|
||||
h := NewHolder()
|
||||
if err := h.Open(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return h
|
||||
}
|
||||
|
||||
// Close closes the holder and removes all underlying data.
|
||||
func (h *Holder) Close() error {
|
||||
defer os.RemoveAll(h.Path)
|
||||
return h.Holder.Close()
|
||||
}
|
||||
|
||||
// MustCreateIndexIfNotExists returns a given index. Panic on error.
|
||||
func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index {
|
||||
idx, err := h.Holder.CreateIndexIfNotExists(index, opt)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &Index{Index: idx}
|
||||
}
|
||||
|
||||
// MustCreateFrameIfNotExists returns a given frame. Panic on error.
|
||||
func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame {
|
||||
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return f
|
||||
}
|
||||
|
||||
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
|
||||
func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
|
||||
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
|
||||
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
v, err := f.CreateViewIfNotExists(view)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
frag, err := v.CreateFragmentIfNotExists(slice)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &Fragment{Fragment: frag}
|
||||
}
|
||||
319
index_test.go
319
index_test.go
|
|
@ -1,262 +1,179 @@
|
|||
package pilosa_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/pilosa/pilosa/pql"
|
||||
)
|
||||
|
||||
// Ensure index can delete a database and its underlying files.
|
||||
func TestIndex_DeleteDB(t *testing.T) {
|
||||
idx := MustOpenIndex()
|
||||
defer idx.Close()
|
||||
// Ensure index can open and retrieve a frame.
|
||||
func TestIndex_CreateFrameIfNotExists(t *testing.T) {
|
||||
index := MustOpenIndex()
|
||||
defer index.Close()
|
||||
|
||||
// Write bits to separate databases.
|
||||
f0 := idx.MustCreateFragmentIfNotExists("d0", "f", pilosa.ViewStandard, 0)
|
||||
if _, err := f0.SetBit(100, 200); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f1 := idx.MustCreateFragmentIfNotExists("d1", "f", pilosa.ViewStandard, 0)
|
||||
if _, err := f1.SetBit(100, 200); err != nil {
|
||||
// Create frame.
|
||||
f, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if f == nil {
|
||||
t.Fatal("expected frame")
|
||||
}
|
||||
|
||||
// Ensure d0 exists.
|
||||
if _, err := os.Stat(idx.DBPath("d0")); err != nil {
|
||||
// Retrieve existing frame.
|
||||
other, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if f.Frame != other.Frame {
|
||||
t.Fatal("frame mismatch")
|
||||
}
|
||||
|
||||
// Delete d0.
|
||||
if err := idx.DeleteDB("d0"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Ensure d0 files are removed & d1 still exists.
|
||||
if _, err := os.Stat(idx.DBPath("d0")); !os.IsNotExist(err) {
|
||||
t.Fatal("expected d0 file deletion")
|
||||
} else if _, err := os.Stat(idx.DBPath("d1")); err != nil {
|
||||
t.Fatal("expected d1 files to still exist", err)
|
||||
if f.Frame != index.Frame("f") {
|
||||
t.Fatal("frame mismatch")
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure index can sync with a remote index.
|
||||
func TestIndexSyncer_SyncIndex(t *testing.T) {
|
||||
cluster := NewCluster(2)
|
||||
// Ensure index defaults the time quantum on new frames.
|
||||
func TestIndex_CreateFrame_TimeQuantum(t *testing.T) {
|
||||
index := MustOpenIndex()
|
||||
defer index.Close()
|
||||
|
||||
// Create a local index.
|
||||
idx0 := MustOpenIndex()
|
||||
defer idx0.Close()
|
||||
|
||||
// Create a remote index wrapped by an HTTP
|
||||
idx1 := MustOpenIndex()
|
||||
defer idx1.Close()
|
||||
s := NewServer()
|
||||
defer s.Close()
|
||||
s.Handler.Index = idx1.Index
|
||||
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
|
||||
e := pilosa.NewExecutor()
|
||||
e.Index = idx1.Index
|
||||
e.Host = cluster.Nodes[1].Host
|
||||
e.Cluster = cluster
|
||||
return e.Execute(ctx, db, query, slices, opt)
|
||||
}
|
||||
|
||||
// Mock 2-node, fully replicated cluster.
|
||||
cluster.ReplicaN = 2
|
||||
cluster.Nodes[0].Host = "localhost:0"
|
||||
cluster.Nodes[1].Host = MustParseURLHost(s.URL)
|
||||
|
||||
// Create frames on nodes.
|
||||
for _, idx := range []*Index{idx0, idx1} {
|
||||
idx.MustCreateFrameIfNotExists("d", "f")
|
||||
idx.MustCreateFrameIfNotExists("d", "f0")
|
||||
idx.MustCreateFrameIfNotExists("y", "z")
|
||||
}
|
||||
|
||||
// Set data on the local index.
|
||||
f := idx0.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
|
||||
if _, err := f.SetBit(0, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(2, 20); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(120, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(200, 4); err != nil {
|
||||
// Set index time quantum.
|
||||
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YM")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f = idx0.MustCreateFragmentIfNotExists("d", "f0", pilosa.ViewStandard, 1)
|
||||
if _, err := f.SetBit(9, SliceWidth+5); err != nil {
|
||||
// Create frame.
|
||||
f, err := index.CreateFrame("f", pilosa.FrameOptions{})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
idx0.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 0)
|
||||
|
||||
// Set data on the remote index.
|
||||
f = idx1.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
|
||||
if _, err := f.SetBit(0, 4000); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(3, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(120, 10); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
f = idx1.MustCreateFragmentIfNotExists("y", "z", pilosa.ViewStandard, 3)
|
||||
if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set highest slice.
|
||||
idx0.DB("d").SetRemoteMaxSlice(1)
|
||||
idx0.DB("y").SetRemoteMaxSlice(3)
|
||||
|
||||
// Set up syncer.
|
||||
syncer := pilosa.IndexSyncer{
|
||||
Index: idx0.Index,
|
||||
Host: cluster.Nodes[0].Host,
|
||||
Cluster: cluster,
|
||||
}
|
||||
|
||||
if err := syncer.SyncIndex(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Verify data is the same on both nodes.
|
||||
for i, idx := range []*Index{idx0, idx1} {
|
||||
f := idx.Fragment("d", "f", pilosa.ViewStandard, 0)
|
||||
if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
|
||||
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
|
||||
} else if a := f.Bitmap(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
|
||||
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
|
||||
} else if a := f.Bitmap(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
|
||||
} else if a := f.Bitmap(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
|
||||
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
|
||||
} else if a := f.Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
|
||||
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
|
||||
}
|
||||
|
||||
f = idx.Fragment("d", "f0", pilosa.ViewStandard, 1)
|
||||
a := f.Bitmap(9).Bits()
|
||||
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
|
||||
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
|
||||
}
|
||||
if a := f.Bitmap(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
|
||||
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
|
||||
}
|
||||
f = idx.Fragment("y", "z", pilosa.ViewStandard, 3)
|
||||
if a := f.Bitmap(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
|
||||
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
|
||||
}
|
||||
} else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YM") {
|
||||
t.Fatalf("unexpected frame time quantum: %s", q)
|
||||
}
|
||||
}
|
||||
|
||||
/* TODO: move this to messenger.go
|
||||
// Ensure index can handle Messenger messages.
|
||||
func TestIndex_HandleMessage(t *testing.T) {
|
||||
// Create a local index.
|
||||
idx0 := MustOpenIndex()
|
||||
defer idx0.Close()
|
||||
// Ensure index can delete a frame.
|
||||
func TestIndex_DeleteFrame(t *testing.T) {
|
||||
index := MustOpenIndex()
|
||||
defer index.Close()
|
||||
|
||||
idx0.MustCreateDBIfNotExists("d", pilosa.DBOptions{})
|
||||
|
||||
msg0 := &internal.CreateSliceMessage{
|
||||
DB: "d",
|
||||
Slice: 8,
|
||||
}
|
||||
idx0.HandleMessage(msg0)
|
||||
if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{"d": 8}) {
|
||||
t.Fatalf("unexpected max slice: %s", ms)
|
||||
// Create frame.
|
||||
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
msg1 := &internal.DeleteDBMessage{
|
||||
DB: "d",
|
||||
// Delete frame & verify it's gone.
|
||||
if err := index.DeleteFrame("f"); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if index.Frame("f") != nil {
|
||||
t.Fatal("expected nil frame")
|
||||
}
|
||||
idx0.HandleMessage(msg1)
|
||||
if ms := idx0.MaxSlices(); !reflect.DeepEqual(ms, map[string]uint64{}) {
|
||||
t.Fatalf("unexpected delete db: %s", ms)
|
||||
|
||||
// Delete again to make sure it doesn't error.
|
||||
if err := index.DeleteFrame("f"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
// Index is a test wrapper for pilosa.Index.
|
||||
// Ensure index can set the default time quantum.
|
||||
func TestIndex_SetTimeQuantum(t *testing.T) {
|
||||
index := MustOpenIndex()
|
||||
defer index.Close()
|
||||
|
||||
// Set & retrieve time quantum.
|
||||
if err := index.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
|
||||
t.Fatalf("unexpected quantum: %s", q)
|
||||
}
|
||||
|
||||
// Reload index and verify that it is persisted.
|
||||
if err := index.Reopen(); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if q := index.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") {
|
||||
t.Fatalf("unexpected quantum (reopen): %s", q)
|
||||
}
|
||||
}
|
||||
|
||||
// Index represents a test wrapper for pilosa.Index.
|
||||
type Index struct {
|
||||
*pilosa.Index
|
||||
LogOutput bytes.Buffer
|
||||
}
|
||||
|
||||
// NewIndex returns a new instance of Index with a temporary path.
|
||||
// NewIndex returns a new instance of Index.
|
||||
func NewIndex() *Index {
|
||||
path, err := ioutil.TempDir("", "pilosa-")
|
||||
path, err := ioutil.TempDir("", "pilosa-index-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
i := &Index{Index: pilosa.NewIndex()}
|
||||
i.Path = path
|
||||
i.Index.LogOutput = &i.LogOutput
|
||||
|
||||
return i
|
||||
}
|
||||
|
||||
// MustOpenIndex creates and opens an index at a temporary path. Panic on error.
|
||||
func MustOpenIndex() *Index {
|
||||
i := NewIndex()
|
||||
if err := i.Open(); err != nil {
|
||||
index, err := pilosa.NewIndex(path, "i")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return i
|
||||
return &Index{Index: index}
|
||||
}
|
||||
|
||||
// Close closes the index and removes all underlying data.
|
||||
// MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
|
||||
func MustOpenIndex() *Index {
|
||||
index := NewIndex()
|
||||
if err := index.Open(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return index
|
||||
}
|
||||
|
||||
// Close closes the index and removes the underlying data.
|
||||
func (i *Index) Close() error {
|
||||
defer os.RemoveAll(i.Path)
|
||||
defer os.RemoveAll(i.Path())
|
||||
return i.Index.Close()
|
||||
}
|
||||
|
||||
// MustCreateDBIfNotExists returns a given db. Panic on error.
|
||||
func (i *Index) MustCreateDBIfNotExists(db string, opt pilosa.DBOptions) *DB {
|
||||
d, err := i.Index.CreateDBIfNotExists(db, opt)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
// Reopen closes the index and reopens it.
|
||||
func (i *Index) Reopen() error {
|
||||
var err error
|
||||
if err := i.Index.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return &DB{DB: d}
|
||||
|
||||
path, name := i.Path(), i.Name()
|
||||
i.Index, err = pilosa.NewIndex(path, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := i.Open(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustCreateFrameIfNotExists returns a given frame. Panic on error.
|
||||
func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame {
|
||||
f, err := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
|
||||
// CreateFrame creates a frame with the given options.
|
||||
func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) {
|
||||
f, err := i.Index.CreateFrame(name, opt)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
return nil, err
|
||||
}
|
||||
return f
|
||||
return &Frame{Frame: f}, nil
|
||||
}
|
||||
|
||||
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
|
||||
func (i *Index) MustCreateFragmentIfNotExists(db, frame, view string, slice uint64) *Fragment {
|
||||
d := i.MustCreateDBIfNotExists(db, pilosa.DBOptions{})
|
||||
f, err := d.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
|
||||
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
|
||||
func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) {
|
||||
f, err := i.Index.CreateFrameIfNotExists(name, opt)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
return nil, err
|
||||
}
|
||||
return &Frame{Frame: f}, nil
|
||||
}
|
||||
|
||||
// Ensure index can delete a frame.
|
||||
func TestIndex_InvalidName(t *testing.T) {
|
||||
path, err := ioutil.TempDir("", "pilosa-index-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
index, err := pilosa.NewIndex(path, "ABC")
|
||||
if index != nil {
|
||||
t.Fatalf("unexpected index name %s", index)
|
||||
}
|
||||
v, err := f.CreateViewIfNotExists(view)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
frag, err := v.CreateFragmentIfNotExists(slice)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &Fragment{Fragment: frag}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -2,9 +2,9 @@ syntax = "proto3";
|
|||
|
||||
package internal;
|
||||
|
||||
message DBMeta {
|
||||
string TimeQuantum = 1;
|
||||
string ColumnLabel = 2;
|
||||
message IndexMeta {
|
||||
string ColumnLabel = 1;
|
||||
string TimeQuantum = 2;
|
||||
}
|
||||
|
||||
message FrameMeta {
|
||||
|
|
@ -20,7 +20,7 @@ message ImportResponse {
|
|||
}
|
||||
|
||||
message BlockDataRequest {
|
||||
string DB = 1;
|
||||
string Index = 1;
|
||||
string Frame = 2;
|
||||
string View = 5;
|
||||
uint64 Slice = 4;
|
||||
|
|
@ -28,12 +28,12 @@ message BlockDataRequest {
|
|||
}
|
||||
|
||||
message BlockDataResponse {
|
||||
repeated uint64 BitmapIDs = 1;
|
||||
repeated uint64 ProfileIDs = 2;
|
||||
repeated uint64 RowIDs = 1;
|
||||
repeated uint64 ColumnIDs = 2;
|
||||
}
|
||||
|
||||
message Cache {
|
||||
repeated uint64 BitmapIDs = 1;
|
||||
repeated uint64 IDs = 1;
|
||||
}
|
||||
|
||||
message MaxSlicesResponse {
|
||||
|
|
@ -41,27 +41,27 @@ message MaxSlicesResponse {
|
|||
}
|
||||
|
||||
message CreateSliceMessage {
|
||||
string DB = 1;
|
||||
string Index = 1;
|
||||
uint64 Slice = 2;
|
||||
}
|
||||
|
||||
message DeleteDBMessage {
|
||||
string DB = 1;
|
||||
message DeleteIndexMessage {
|
||||
string Index = 1;
|
||||
}
|
||||
|
||||
message CreateDBMessage {
|
||||
string DB = 1;
|
||||
DBMeta Meta = 2;
|
||||
message CreateIndexMessage {
|
||||
string Index = 1;
|
||||
IndexMeta Meta = 2;
|
||||
}
|
||||
|
||||
message CreateFrameMessage {
|
||||
string DB = 1;
|
||||
string Index = 1;
|
||||
string Frame = 2;
|
||||
FrameMeta Meta = 3;
|
||||
}
|
||||
|
||||
message DeleteFrameMessage {
|
||||
string DB = 1;
|
||||
string Index = 1;
|
||||
string Frame = 2;
|
||||
}
|
||||
|
||||
|
|
@ -70,15 +70,19 @@ message Frame {
|
|||
FrameMeta Meta = 2;
|
||||
}
|
||||
|
||||
message DB {
|
||||
message Index {
|
||||
string Name = 1;
|
||||
DBMeta Meta = 2;
|
||||
IndexMeta Meta = 2;
|
||||
uint64 MaxSlice = 3;
|
||||
repeated Frame Frames = 4;
|
||||
}
|
||||
|
||||
message NodeState {
|
||||
message NodeStatus {
|
||||
string Host = 1;
|
||||
string State = 2;
|
||||
repeated DB DBs = 3;
|
||||
repeated Index Indexes = 3;
|
||||
}
|
||||
|
||||
message ClusterStatus {
|
||||
repeated NodeStatus Nodes = 1;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
Bitmap
|
||||
Pair
|
||||
Bit
|
||||
Profile
|
||||
ColumnAttrSet
|
||||
Attr
|
||||
AttrMap
|
||||
QueryRequest
|
||||
|
|
@ -67,8 +67,8 @@ func (*Pair) ProtoMessage() {}
|
|||
func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} }
|
||||
|
||||
type Bit struct {
|
||||
BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,proto3" json:"BitmapID,omitempty"`
|
||||
ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,proto3" json:"ProfileID,omitempty"`
|
||||
RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"`
|
||||
ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"`
|
||||
Timestamp int64 `protobuf:"varint,3,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"`
|
||||
}
|
||||
|
||||
|
|
@ -77,17 +77,17 @@ func (m *Bit) String() string { return proto.CompactTextString(m) }
|
|||
func (*Bit) ProtoMessage() {}
|
||||
func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
|
||||
|
||||
type Profile struct {
|
||||
type ColumnAttrSet struct {
|
||||
ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
|
||||
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
|
||||
}
|
||||
|
||||
func (m *Profile) Reset() { *m = Profile{} }
|
||||
func (m *Profile) String() string { return proto.CompactTextString(m) }
|
||||
func (*Profile) ProtoMessage() {}
|
||||
func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} }
|
||||
func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} }
|
||||
func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) }
|
||||
func (*ColumnAttrSet) ProtoMessage() {}
|
||||
func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} }
|
||||
|
||||
func (m *Profile) GetAttrs() []*Attr {
|
||||
func (m *ColumnAttrSet) GetAttrs() []*Attr {
|
||||
if m != nil {
|
||||
return m.Attrs
|
||||
}
|
||||
|
|
@ -125,11 +125,11 @@ func (m *AttrMap) GetAttrs() []*Attr {
|
|||
}
|
||||
|
||||
type QueryRequest struct {
|
||||
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
|
||||
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
|
||||
Profiles bool `protobuf:"varint,3,opt,name=Profiles,proto3" json:"Profiles,omitempty"`
|
||||
Quantum string `protobuf:"bytes,4,opt,name=Quantum,proto3" json:"Quantum,omitempty"`
|
||||
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
|
||||
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
|
||||
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
|
||||
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
|
||||
Quantum string `protobuf:"bytes,4,opt,name=Quantum,proto3" json:"Quantum,omitempty"`
|
||||
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
|
||||
}
|
||||
|
||||
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
|
||||
|
|
@ -138,9 +138,9 @@ func (*QueryRequest) ProtoMessage() {}
|
|||
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} }
|
||||
|
||||
type QueryResponse struct {
|
||||
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
|
||||
Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"`
|
||||
Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"`
|
||||
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
|
||||
Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"`
|
||||
ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"`
|
||||
}
|
||||
|
||||
func (m *QueryResponse) Reset() { *m = QueryResponse{} }
|
||||
|
|
@ -155,9 +155,9 @@ func (m *QueryResponse) GetResults() []*QueryResult {
|
|||
return nil
|
||||
}
|
||||
|
||||
func (m *QueryResponse) GetProfiles() []*Profile {
|
||||
func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet {
|
||||
if m != nil {
|
||||
return m.Profiles
|
||||
return m.ColumnAttrSets
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -189,11 +189,11 @@ func (m *QueryResult) GetPairs() []*Pair {
|
|||
}
|
||||
|
||||
type ImportRequest struct {
|
||||
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
|
||||
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
|
||||
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
|
||||
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
|
||||
BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"`
|
||||
ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"`
|
||||
RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"`
|
||||
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
|
||||
Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"`
|
||||
}
|
||||
|
||||
|
|
@ -206,7 +206,7 @@ func init() {
|
|||
proto.RegisterType((*Bitmap)(nil), "internal.Bitmap")
|
||||
proto.RegisterType((*Pair)(nil), "internal.Pair")
|
||||
proto.RegisterType((*Bit)(nil), "internal.Bit")
|
||||
proto.RegisterType((*Profile)(nil), "internal.Profile")
|
||||
proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet")
|
||||
proto.RegisterType((*Attr)(nil), "internal.Attr")
|
||||
proto.RegisterType((*AttrMap)(nil), "internal.AttrMap")
|
||||
proto.RegisterType((*QueryRequest)(nil), "internal.QueryRequest")
|
||||
|
|
@ -304,15 +304,15 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) {
|
|||
_ = i
|
||||
var l int
|
||||
_ = l
|
||||
if m.BitmapID != 0 {
|
||||
if m.RowID != 0 {
|
||||
dAtA[i] = 0x8
|
||||
i++
|
||||
i = encodeVarintPublic(dAtA, i, uint64(m.BitmapID))
|
||||
i = encodeVarintPublic(dAtA, i, uint64(m.RowID))
|
||||
}
|
||||
if m.ProfileID != 0 {
|
||||
if m.ColumnID != 0 {
|
||||
dAtA[i] = 0x10
|
||||
i++
|
||||
i = encodeVarintPublic(dAtA, i, uint64(m.ProfileID))
|
||||
i = encodeVarintPublic(dAtA, i, uint64(m.ColumnID))
|
||||
}
|
||||
if m.Timestamp != 0 {
|
||||
dAtA[i] = 0x18
|
||||
|
|
@ -322,7 +322,7 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) {
|
|||
return i, nil
|
||||
}
|
||||
|
||||
func (m *Profile) Marshal() (dAtA []byte, err error) {
|
||||
func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) {
|
||||
size := m.Size()
|
||||
dAtA = make([]byte, size)
|
||||
n, err := m.MarshalTo(dAtA)
|
||||
|
|
@ -332,7 +332,7 @@ func (m *Profile) Marshal() (dAtA []byte, err error) {
|
|||
return dAtA[:n], nil
|
||||
}
|
||||
|
||||
func (m *Profile) MarshalTo(dAtA []byte) (int, error) {
|
||||
func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) {
|
||||
var i int
|
||||
_ = i
|
||||
var l int
|
||||
|
|
@ -480,10 +480,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) {
|
|||
i = encodeVarintPublic(dAtA, i, uint64(j3))
|
||||
i += copy(dAtA[i:], dAtA4[:j3])
|
||||
}
|
||||
if m.Profiles {
|
||||
if m.ColumnAttrs {
|
||||
dAtA[i] = 0x18
|
||||
i++
|
||||
if m.Profiles {
|
||||
if m.ColumnAttrs {
|
||||
dAtA[i] = 1
|
||||
} else {
|
||||
dAtA[i] = 0
|
||||
|
|
@ -542,8 +542,8 @@ func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) {
|
|||
i += n
|
||||
}
|
||||
}
|
||||
if len(m.Profiles) > 0 {
|
||||
for _, msg := range m.Profiles {
|
||||
if len(m.ColumnAttrSets) > 0 {
|
||||
for _, msg := range m.ColumnAttrSets {
|
||||
dAtA[i] = 0x1a
|
||||
i++
|
||||
i = encodeVarintPublic(dAtA, i, uint64(msg.Size()))
|
||||
|
|
@ -627,11 +627,11 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
|
|||
_ = i
|
||||
var l int
|
||||
_ = l
|
||||
if len(m.DB) > 0 {
|
||||
if len(m.Index) > 0 {
|
||||
dAtA[i] = 0xa
|
||||
i++
|
||||
i = encodeVarintPublic(dAtA, i, uint64(len(m.DB)))
|
||||
i += copy(dAtA[i:], m.DB)
|
||||
i = encodeVarintPublic(dAtA, i, uint64(len(m.Index)))
|
||||
i += copy(dAtA[i:], m.Index)
|
||||
}
|
||||
if len(m.Frame) > 0 {
|
||||
dAtA[i] = 0x12
|
||||
|
|
@ -644,10 +644,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
|
|||
i++
|
||||
i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
|
||||
}
|
||||
if len(m.BitmapIDs) > 0 {
|
||||
dAtA7 := make([]byte, len(m.BitmapIDs)*10)
|
||||
if len(m.RowIDs) > 0 {
|
||||
dAtA7 := make([]byte, len(m.RowIDs)*10)
|
||||
var j6 int
|
||||
for _, num := range m.BitmapIDs {
|
||||
for _, num := range m.RowIDs {
|
||||
for num >= 1<<7 {
|
||||
dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80)
|
||||
num >>= 7
|
||||
|
|
@ -661,10 +661,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
|
|||
i = encodeVarintPublic(dAtA, i, uint64(j6))
|
||||
i += copy(dAtA[i:], dAtA7[:j6])
|
||||
}
|
||||
if len(m.ProfileIDs) > 0 {
|
||||
dAtA9 := make([]byte, len(m.ProfileIDs)*10)
|
||||
if len(m.ColumnIDs) > 0 {
|
||||
dAtA9 := make([]byte, len(m.ColumnIDs)*10)
|
||||
var j8 int
|
||||
for _, num := range m.ProfileIDs {
|
||||
for _, num := range m.ColumnIDs {
|
||||
for num >= 1<<7 {
|
||||
dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80)
|
||||
num >>= 7
|
||||
|
|
@ -760,11 +760,11 @@ func (m *Pair) Size() (n int) {
|
|||
func (m *Bit) Size() (n int) {
|
||||
var l int
|
||||
_ = l
|
||||
if m.BitmapID != 0 {
|
||||
n += 1 + sovPublic(uint64(m.BitmapID))
|
||||
if m.RowID != 0 {
|
||||
n += 1 + sovPublic(uint64(m.RowID))
|
||||
}
|
||||
if m.ProfileID != 0 {
|
||||
n += 1 + sovPublic(uint64(m.ProfileID))
|
||||
if m.ColumnID != 0 {
|
||||
n += 1 + sovPublic(uint64(m.ColumnID))
|
||||
}
|
||||
if m.Timestamp != 0 {
|
||||
n += 1 + sovPublic(uint64(m.Timestamp))
|
||||
|
|
@ -772,7 +772,7 @@ func (m *Bit) Size() (n int) {
|
|||
return n
|
||||
}
|
||||
|
||||
func (m *Profile) Size() (n int) {
|
||||
func (m *ColumnAttrSet) Size() (n int) {
|
||||
var l int
|
||||
_ = l
|
||||
if m.ID != 0 {
|
||||
|
|
@ -839,7 +839,7 @@ func (m *QueryRequest) Size() (n int) {
|
|||
}
|
||||
n += 1 + sovPublic(uint64(l)) + l
|
||||
}
|
||||
if m.Profiles {
|
||||
if m.ColumnAttrs {
|
||||
n += 2
|
||||
}
|
||||
l = len(m.Quantum)
|
||||
|
|
@ -865,8 +865,8 @@ func (m *QueryResponse) Size() (n int) {
|
|||
n += 1 + l + sovPublic(uint64(l))
|
||||
}
|
||||
}
|
||||
if len(m.Profiles) > 0 {
|
||||
for _, e := range m.Profiles {
|
||||
if len(m.ColumnAttrSets) > 0 {
|
||||
for _, e := range m.ColumnAttrSets {
|
||||
l = e.Size()
|
||||
n += 1 + l + sovPublic(uint64(l))
|
||||
}
|
||||
|
|
@ -899,7 +899,7 @@ func (m *QueryResult) Size() (n int) {
|
|||
func (m *ImportRequest) Size() (n int) {
|
||||
var l int
|
||||
_ = l
|
||||
l = len(m.DB)
|
||||
l = len(m.Index)
|
||||
if l > 0 {
|
||||
n += 1 + l + sovPublic(uint64(l))
|
||||
}
|
||||
|
|
@ -910,16 +910,16 @@ func (m *ImportRequest) Size() (n int) {
|
|||
if m.Slice != 0 {
|
||||
n += 1 + sovPublic(uint64(m.Slice))
|
||||
}
|
||||
if len(m.BitmapIDs) > 0 {
|
||||
if len(m.RowIDs) > 0 {
|
||||
l = 0
|
||||
for _, e := range m.BitmapIDs {
|
||||
for _, e := range m.RowIDs {
|
||||
l += sovPublic(uint64(e))
|
||||
}
|
||||
n += 1 + sovPublic(uint64(l)) + l
|
||||
}
|
||||
if len(m.ProfileIDs) > 0 {
|
||||
if len(m.ColumnIDs) > 0 {
|
||||
l = 0
|
||||
for _, e := range m.ProfileIDs {
|
||||
for _, e := range m.ColumnIDs {
|
||||
l += sovPublic(uint64(e))
|
||||
}
|
||||
n += 1 + sovPublic(uint64(l)) + l
|
||||
|
|
@ -1209,9 +1209,9 @@ func (m *Bit) Unmarshal(dAtA []byte) error {
|
|||
switch fieldNum {
|
||||
case 1:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field BitmapID", wireType)
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field RowID", wireType)
|
||||
}
|
||||
m.BitmapID = 0
|
||||
m.RowID = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowPublic
|
||||
|
|
@ -1221,16 +1221,16 @@ func (m *Bit) Unmarshal(dAtA []byte) error {
|
|||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.BitmapID |= (uint64(b) & 0x7F) << shift
|
||||
m.RowID |= (uint64(b) & 0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
}
|
||||
case 2:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field ProfileID", wireType)
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field ColumnID", wireType)
|
||||
}
|
||||
m.ProfileID = 0
|
||||
m.ColumnID = 0
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
if shift >= 64 {
|
||||
return ErrIntOverflowPublic
|
||||
|
|
@ -1240,7 +1240,7 @@ func (m *Bit) Unmarshal(dAtA []byte) error {
|
|||
}
|
||||
b := dAtA[iNdEx]
|
||||
iNdEx++
|
||||
m.ProfileID |= (uint64(b) & 0x7F) << shift
|
||||
m.ColumnID |= (uint64(b) & 0x7F) << shift
|
||||
if b < 0x80 {
|
||||
break
|
||||
}
|
||||
|
|
@ -1285,7 +1285,7 @@ func (m *Bit) Unmarshal(dAtA []byte) error {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
func (m *Profile) Unmarshal(dAtA []byte) error {
|
||||
func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error {
|
||||
l := len(dAtA)
|
||||
iNdEx := 0
|
||||
for iNdEx < l {
|
||||
|
|
@ -1308,10 +1308,10 @@ func (m *Profile) Unmarshal(dAtA []byte) error {
|
|||
fieldNum := int32(wire >> 3)
|
||||
wireType := int(wire & 0x7)
|
||||
if wireType == 4 {
|
||||
return fmt.Errorf("proto: Profile: wiretype end group for non-group")
|
||||
return fmt.Errorf("proto: ColumnAttrSet: wiretype end group for non-group")
|
||||
}
|
||||
if fieldNum <= 0 {
|
||||
return fmt.Errorf("proto: Profile: illegal tag %d (wire type %d)", fieldNum, wire)
|
||||
return fmt.Errorf("proto: ColumnAttrSet: illegal tag %d (wire type %d)", fieldNum, wire)
|
||||
}
|
||||
switch fieldNum {
|
||||
case 1:
|
||||
|
|
@ -1772,7 +1772,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
|
|||
}
|
||||
case 3:
|
||||
if wireType != 0 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType)
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field ColumnAttrs", wireType)
|
||||
}
|
||||
var v int
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
|
|
@ -1789,7 +1789,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
|
|||
break
|
||||
}
|
||||
}
|
||||
m.Profiles = bool(v != 0)
|
||||
m.ColumnAttrs = bool(v != 0)
|
||||
case 4:
|
||||
if wireType != 2 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Quantum", wireType)
|
||||
|
|
@ -1951,7 +1951,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error {
|
|||
iNdEx = postIndex
|
||||
case 3:
|
||||
if wireType != 2 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType)
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field ColumnAttrSets", wireType)
|
||||
}
|
||||
var msglen int
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
|
|
@ -1975,8 +1975,8 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error {
|
|||
if postIndex > l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
m.Profiles = append(m.Profiles, &Profile{})
|
||||
if err := m.Profiles[len(m.Profiles)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
|
||||
m.ColumnAttrSets = append(m.ColumnAttrSets, &ColumnAttrSet{})
|
||||
if err := m.ColumnAttrSets[len(m.ColumnAttrSets)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
|
||||
return err
|
||||
}
|
||||
iNdEx = postIndex
|
||||
|
|
@ -2185,7 +2185,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
|
|||
switch fieldNum {
|
||||
case 1:
|
||||
if wireType != 2 {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType)
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field Index", wireType)
|
||||
}
|
||||
var stringLen uint64
|
||||
for shift := uint(0); ; shift += 7 {
|
||||
|
|
@ -2210,7 +2210,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
|
|||
if postIndex > l {
|
||||
return io.ErrUnexpectedEOF
|
||||
}
|
||||
m.DB = string(dAtA[iNdEx:postIndex])
|
||||
m.Index = string(dAtA[iNdEx:postIndex])
|
||||
iNdEx = postIndex
|
||||
case 2:
|
||||
if wireType != 2 {
|
||||
|
|
@ -2300,7 +2300,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
|
|||
break
|
||||
}
|
||||
}
|
||||
m.BitmapIDs = append(m.BitmapIDs, v)
|
||||
m.RowIDs = append(m.RowIDs, v)
|
||||
}
|
||||
} else if wireType == 0 {
|
||||
var v uint64
|
||||
|
|
@ -2318,9 +2318,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
|
|||
break
|
||||
}
|
||||
}
|
||||
m.BitmapIDs = append(m.BitmapIDs, v)
|
||||
m.RowIDs = append(m.RowIDs, v)
|
||||
} else {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType)
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType)
|
||||
}
|
||||
case 5:
|
||||
if wireType == 2 {
|
||||
|
|
@ -2362,7 +2362,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
|
|||
break
|
||||
}
|
||||
}
|
||||
m.ProfileIDs = append(m.ProfileIDs, v)
|
||||
m.ColumnIDs = append(m.ColumnIDs, v)
|
||||
}
|
||||
} else if wireType == 0 {
|
||||
var v uint64
|
||||
|
|
@ -2380,9 +2380,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
|
|||
break
|
||||
}
|
||||
}
|
||||
m.ProfileIDs = append(m.ProfileIDs, v)
|
||||
m.ColumnIDs = append(m.ColumnIDs, v)
|
||||
} else {
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType)
|
||||
return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType)
|
||||
}
|
||||
case 6:
|
||||
if wireType == 2 {
|
||||
|
|
@ -2575,41 +2575,41 @@ var (
|
|||
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
|
||||
|
||||
var fileDescriptorPublic = []byte{
|
||||
// 570 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x6e, 0xd4, 0x40,
|
||||
0x10, 0xa5, 0x6d, 0xcf, 0xaf, 0x26, 0x19, 0x0d, 0x2d, 0x40, 0x16, 0x42, 0x23, 0xcb, 0x62, 0xe1,
|
||||
0x0d, 0x13, 0x29, 0x1c, 0x00, 0xe1, 0x4c, 0x22, 0x8d, 0x10, 0x51, 0xd2, 0x89, 0xd8, 0xb1, 0x70,
|
||||
0x42, 0x13, 0x2c, 0xf9, 0x47, 0x77, 0x7b, 0x31, 0x4b, 0x16, 0x6c, 0x38, 0x01, 0x47, 0x80, 0x9b,
|
||||
0xb0, 0xe4, 0x08, 0x68, 0xb8, 0x08, 0xaa, 0xfe, 0xd8, 0x66, 0x83, 0xd8, 0xf5, 0x7b, 0xe5, 0xea,
|
||||
0xae, 0xf7, 0xaa, 0xca, 0x70, 0xd0, 0xb4, 0x37, 0x45, 0x7e, 0xbb, 0x6e, 0x44, 0xad, 0x6a, 0x3a,
|
||||
0xcd, 0x2b, 0xc5, 0x45, 0x95, 0x15, 0x71, 0x0a, 0xe3, 0x34, 0x57, 0x65, 0xd6, 0x50, 0x0a, 0x41,
|
||||
0x9a, 0x2b, 0x19, 0x92, 0xc8, 0x4f, 0x02, 0xa6, 0xcf, 0xf4, 0x29, 0x8c, 0x5e, 0x2a, 0x25, 0x64,
|
||||
0xe8, 0x45, 0x7e, 0x32, 0x3f, 0x5e, 0xac, 0x5d, 0xde, 0x1a, 0x69, 0x66, 0x82, 0xf1, 0x1a, 0x82,
|
||||
0x8b, 0x2c, 0x17, 0x74, 0x09, 0xfe, 0x2b, 0xbe, 0x0b, 0x49, 0x44, 0x92, 0x80, 0xe1, 0x91, 0x3e,
|
||||
0x80, 0xd1, 0x49, 0xdd, 0x56, 0x2a, 0xf4, 0x34, 0x67, 0x40, 0xfc, 0x16, 0xfc, 0x34, 0x57, 0xf4,
|
||||
0x31, 0x4c, 0xcd, 0xd3, 0xdb, 0x8d, 0xcd, 0xe9, 0x30, 0x7d, 0x02, 0xb3, 0x0b, 0x51, 0xbf, 0xcf,
|
||||
0x0b, 0xbe, 0xdd, 0xd8, 0xe4, 0x9e, 0xc0, 0xe8, 0x75, 0x5e, 0x72, 0xa9, 0xb2, 0xb2, 0x09, 0xfd,
|
||||
0x88, 0x24, 0x3e, 0xeb, 0x89, 0xf8, 0x05, 0x4c, 0xec, 0xa7, 0x74, 0x01, 0x5e, 0x77, 0xb9, 0xb7,
|
||||
0xdd, 0xfc, 0xa7, 0x9e, 0x6f, 0x04, 0x02, 0x3c, 0x0d, 0x05, 0xcd, 0x8c, 0x20, 0x0a, 0xc1, 0xf5,
|
||||
0xae, 0xe1, 0xb6, 0x24, 0x7d, 0xa6, 0x11, 0xcc, 0xaf, 0x94, 0xc8, 0xab, 0xbb, 0x37, 0x59, 0xd1,
|
||||
0x72, 0x5d, 0xcf, 0x8c, 0x0d, 0x29, 0x54, 0xba, 0xad, 0x94, 0x09, 0x07, 0xba, 0xdc, 0x0e, 0xa3,
|
||||
0x96, 0xb4, 0xae, 0x0b, 0x13, 0x1c, 0x45, 0x24, 0x99, 0xb2, 0x9e, 0xa0, 0x2b, 0x80, 0xb3, 0xa2,
|
||||
0xce, 0x6c, 0xee, 0x38, 0x22, 0x09, 0x61, 0x03, 0x26, 0x3e, 0x82, 0x09, 0x56, 0xfa, 0x3a, 0x6b,
|
||||
0x7a, 0x6d, 0xe4, 0x5f, 0xda, 0xbe, 0x10, 0x38, 0xb8, 0x6c, 0xb9, 0xd8, 0x31, 0xfe, 0xb1, 0xe5,
|
||||
0x52, 0x61, 0x8b, 0x34, 0xb6, 0x2a, 0x0d, 0xa0, 0x8f, 0x60, 0x7c, 0x55, 0xe4, 0xb7, 0xdc, 0x38,
|
||||
0x15, 0x30, 0x8b, 0x50, 0x89, 0xf5, 0x56, 0x6a, 0xa1, 0x53, 0xd6, 0x61, 0x1a, 0xc2, 0xe4, 0xb2,
|
||||
0xcd, 0x2a, 0xd5, 0x96, 0x5a, 0xe4, 0x8c, 0x39, 0x88, 0xb7, 0x31, 0x5e, 0xd6, 0xca, 0x09, 0xb4,
|
||||
0x28, 0xfe, 0x44, 0xe0, 0xd0, 0x16, 0x23, 0x9b, 0xba, 0x92, 0x1c, 0x1d, 0x3f, 0x15, 0xc2, 0x39,
|
||||
0x7e, 0x2a, 0x04, 0x3d, 0x82, 0x09, 0xe3, 0xb2, 0x2d, 0x94, 0x6b, 0xda, 0xc3, 0x5e, 0x98, 0xcb,
|
||||
0x6d, 0x0b, 0xc5, 0xdc, 0x57, 0xf4, 0xd9, 0x5f, 0x25, 0x62, 0xc6, 0xfd, 0x3e, 0xc3, 0x46, 0xfa,
|
||||
0xaa, 0xe3, 0xcf, 0x04, 0xe6, 0x83, 0x7b, 0x68, 0xe2, 0x16, 0x42, 0x17, 0x31, 0x3f, 0x5e, 0xf6,
|
||||
0xc9, 0x86, 0x67, 0x6e, 0x61, 0x0e, 0x80, 0x9c, 0xdb, 0x41, 0x20, 0xe7, 0x68, 0x3f, 0x2e, 0x81,
|
||||
0x7b, 0x73, 0x60, 0x3f, 0xd2, 0xcc, 0x04, 0xd1, 0xa3, 0x93, 0x0f, 0x59, 0x75, 0xc7, 0xdf, 0x69,
|
||||
0x8f, 0xa6, 0xcc, 0xc1, 0xf8, 0x3b, 0x81, 0xc3, 0x6d, 0xd9, 0xd4, 0x42, 0xb9, 0xce, 0x2c, 0xc0,
|
||||
0xdb, 0xa4, 0xd6, 0x0a, 0x6f, 0x93, 0x62, 0xa7, 0xce, 0x44, 0x56, 0x9a, 0xe1, 0x9b, 0x31, 0x03,
|
||||
0x90, 0xd5, 0xbd, 0xd1, 0xed, 0x08, 0x98, 0x01, 0x7a, 0xaa, 0xec, 0x2e, 0xc9, 0x30, 0xd0, 0x2d,
|
||||
0xec, 0x09, 0x9c, 0xaa, 0x6e, 0x99, 0x64, 0x38, 0xd2, 0xe1, 0x01, 0x83, 0xf1, 0x6e, 0x9d, 0x64,
|
||||
0x38, 0x8e, 0xfc, 0xc4, 0x67, 0x03, 0x26, 0x5d, 0xfe, 0xd8, 0xaf, 0xc8, 0xcf, 0xfd, 0x8a, 0xfc,
|
||||
0xda, 0xaf, 0xc8, 0xd7, 0xdf, 0xab, 0x7b, 0x37, 0x63, 0xfd, 0x5f, 0x79, 0xfe, 0x27, 0x00, 0x00,
|
||||
0xff, 0xff, 0x37, 0xb6, 0x15, 0x22, 0x67, 0x04, 0x00, 0x00,
|
||||
// 576 bytes of a gzipped FileDescriptorProto
|
||||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40,
|
||||
0x10, 0xa5, 0x63, 0xe7, 0x57, 0xf9, 0x28, 0x6a, 0xf1, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x85, 0x57,
|
||||
0x19, 0x69, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xa6, 0x33, 0xb0, 0xf7, 0xcc, 0xb4,
|
||||
0x06, 0x4b, 0xfe, 0xd1, 0xdd, 0x16, 0xe4, 0x00, 0xec, 0x91, 0xd8, 0x70, 0x03, 0x38, 0x0a, 0x4b,
|
||||
0x8e, 0x80, 0xc2, 0x45, 0x50, 0x75, 0xbb, 0x63, 0x0f, 0x0b, 0xc4, 0xae, 0xdf, 0xab, 0xae, 0x76,
|
||||
0xbd, 0x7a, 0x55, 0x86, 0x69, 0x55, 0x5f, 0x65, 0xe9, 0xf5, 0xaa, 0x12, 0xa5, 0x2a, 0xe9, 0x28,
|
||||
0x2d, 0x14, 0x17, 0x45, 0x92, 0x05, 0x11, 0x0c, 0xa2, 0x54, 0xe5, 0x49, 0x45, 0x29, 0xb8, 0x51,
|
||||
0xaa, 0xa4, 0x47, 0x7c, 0x27, 0x74, 0x99, 0x3e, 0xd3, 0xa7, 0xd0, 0x7f, 0xa1, 0x94, 0x90, 0x5e,
|
||||
0xcf, 0x77, 0xc2, 0xc9, 0xe9, 0x7c, 0x65, 0xf3, 0x56, 0x48, 0x33, 0x13, 0x0c, 0x56, 0xe0, 0xbe,
|
||||
0x4e, 0x52, 0x41, 0x17, 0xe0, 0xbc, 0xe4, 0x7b, 0x8f, 0xf8, 0x24, 0x74, 0x19, 0x1e, 0xe9, 0x7d,
|
||||
0xe8, 0xaf, 0xcb, 0xba, 0x50, 0x5e, 0x4f, 0x73, 0x06, 0x04, 0x6f, 0xc0, 0x89, 0x52, 0x85, 0x41,
|
||||
0x56, 0x7e, 0x88, 0x37, 0x4d, 0x82, 0x01, 0xf4, 0x31, 0x8c, 0xd6, 0x65, 0x56, 0xe7, 0x45, 0xbc,
|
||||
0x69, 0xb2, 0x8e, 0x98, 0x3e, 0x81, 0xf1, 0x65, 0x9a, 0x73, 0xa9, 0x92, 0xbc, 0xf2, 0x1c, 0x9f,
|
||||
0x84, 0x0e, 0x6b, 0x89, 0x60, 0x0b, 0x33, 0x73, 0x13, 0xab, 0xda, 0x71, 0x45, 0xe7, 0xd0, 0x3b,
|
||||
0xbe, 0xde, 0x8b, 0x37, 0xff, 0xa9, 0xe6, 0x3b, 0x01, 0x17, 0x4f, 0x5d, 0x39, 0x63, 0x23, 0x87,
|
||||
0x82, 0x7b, 0xb9, 0xaf, 0x78, 0x53, 0x97, 0x3e, 0x53, 0x1f, 0x26, 0x3b, 0x25, 0xd2, 0xe2, 0xf6,
|
||||
0x6d, 0x92, 0xd5, 0x5c, 0x57, 0x35, 0x66, 0x5d, 0x0a, 0x15, 0xc5, 0x85, 0x32, 0x61, 0x57, 0x17,
|
||||
0x7d, 0xc4, 0xa8, 0x28, 0x2a, 0xcb, 0xcc, 0x04, 0xfb, 0x3e, 0x09, 0x47, 0xac, 0x25, 0xe8, 0x12,
|
||||
0xe0, 0x2c, 0x2b, 0x93, 0x26, 0x77, 0xe0, 0x93, 0x90, 0xb0, 0x0e, 0x13, 0x9c, 0xc0, 0x10, 0x2b,
|
||||
0x7d, 0x95, 0x54, 0xad, 0x36, 0xf2, 0x2f, 0x6d, 0x9f, 0x09, 0x4c, 0x2f, 0x6a, 0x2e, 0xf6, 0x8c,
|
||||
0xbf, 0xaf, 0xb9, 0xd4, 0x1e, 0x68, 0xdc, 0xa8, 0x34, 0x80, 0x3e, 0x84, 0xc1, 0x2e, 0x4b, 0xaf,
|
||||
0xb9, 0xe9, 0x94, 0xcb, 0x1a, 0x84, 0x5a, 0xdb, 0x0e, 0x4b, 0xad, 0x75, 0xc4, 0xba, 0x14, 0xf5,
|
||||
0x60, 0x78, 0x51, 0x27, 0x85, 0xaa, 0x73, 0x2d, 0x75, 0xcc, 0x2c, 0xc4, 0x37, 0x19, 0xcf, 0x4b,
|
||||
0x65, 0x65, 0x36, 0x28, 0xf8, 0x42, 0x60, 0xd6, 0x94, 0x24, 0xab, 0xb2, 0x90, 0x1c, 0xfb, 0xbe,
|
||||
0x15, 0xc2, 0xf6, 0x7d, 0x2b, 0x04, 0x3d, 0x81, 0x21, 0xe3, 0xb2, 0xce, 0x94, 0xb5, 0xee, 0x41,
|
||||
0x2b, 0xcf, 0xe6, 0xd6, 0x99, 0x62, 0xf6, 0x16, 0x7d, 0x0e, 0xf3, 0x3b, 0xa3, 0x80, 0xb5, 0x62,
|
||||
0xde, 0xa3, 0x36, 0xef, 0x4e, 0x9c, 0xfd, 0x75, 0x3d, 0xf8, 0x44, 0x60, 0xd2, 0x79, 0x99, 0x86,
|
||||
0x76, 0x4d, 0x74, 0x59, 0x93, 0xd3, 0x45, 0xfb, 0x90, 0xe1, 0x99, 0x5d, 0xa3, 0x29, 0x90, 0xf3,
|
||||
0x66, 0x40, 0xc8, 0x39, 0xda, 0x82, 0xab, 0x61, 0xbf, 0xdf, 0xb1, 0x05, 0x69, 0x66, 0x82, 0xd8,
|
||||
0xb5, 0xf5, 0xbb, 0xa4, 0xb8, 0xe5, 0x37, 0xba, 0x6b, 0x23, 0x66, 0x61, 0xf0, 0x8d, 0xc0, 0x2c,
|
||||
0xce, 0xab, 0x52, 0xa8, 0x8e, 0x63, 0x71, 0x71, 0xc3, 0x3f, 0x5a, 0xc7, 0x34, 0x40, 0xf6, 0x4c,
|
||||
0x24, 0xb9, 0x19, 0xcd, 0x31, 0x33, 0x00, 0x59, 0xed, 0x9c, 0x76, 0xca, 0x65, 0x06, 0x68, 0x27,
|
||||
0x70, 0xd5, 0xa4, 0xe7, 0x1a, 0x77, 0x0d, 0xc2, 0x59, 0xb4, 0x9b, 0x26, 0xbd, 0xbe, 0x0e, 0xb5,
|
||||
0x04, 0xce, 0xe2, 0x71, 0xd5, 0xa4, 0x37, 0xf0, 0x9d, 0xd0, 0x61, 0x1d, 0x26, 0x5a, 0xfc, 0x38,
|
||||
0x2c, 0xc9, 0xcf, 0xc3, 0x92, 0xfc, 0x3a, 0x2c, 0xc9, 0xd7, 0xdf, 0xcb, 0x7b, 0x57, 0x03, 0xfd,
|
||||
0xaf, 0x79, 0xf6, 0x27, 0x00, 0x00, 0xff, 0xff, 0x48, 0x20, 0x4b, 0xe7, 0x7b, 0x04, 0x00, 0x00,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,12 +13,12 @@ message Pair {
|
|||
}
|
||||
|
||||
message Bit {
|
||||
uint64 BitmapID = 1;
|
||||
uint64 ProfileID = 2;
|
||||
uint64 RowID = 1;
|
||||
uint64 ColumnID = 2;
|
||||
int64 Timestamp = 3;
|
||||
}
|
||||
|
||||
message Profile {
|
||||
message ColumnAttrSet {
|
||||
uint64 ID = 1;
|
||||
repeated Attr Attrs = 2;
|
||||
}
|
||||
|
|
@ -39,7 +39,7 @@ message AttrMap {
|
|||
message QueryRequest {
|
||||
string Query = 1;
|
||||
repeated uint64 Slices = 2;
|
||||
bool Profiles = 3;
|
||||
bool ColumnAttrs = 3;
|
||||
string Quantum = 4;
|
||||
bool Remote = 5;
|
||||
}
|
||||
|
|
@ -47,7 +47,7 @@ message QueryRequest {
|
|||
message QueryResponse {
|
||||
string Err = 1;
|
||||
repeated QueryResult Results = 2;
|
||||
repeated Profile Profiles = 3;
|
||||
repeated ColumnAttrSet ColumnAttrSets = 3;
|
||||
}
|
||||
|
||||
message QueryResult {
|
||||
|
|
@ -58,10 +58,10 @@ message QueryResult {
|
|||
}
|
||||
|
||||
message ImportRequest {
|
||||
string DB = 1;
|
||||
string Index = 1;
|
||||
string Frame = 2;
|
||||
uint64 Slice = 3;
|
||||
repeated uint64 BitmapIDs = 4;
|
||||
repeated uint64 ProfileIDs = 5;
|
||||
repeated uint64 RowIDs = 4;
|
||||
repeated uint64 ColumnIDs = 5;
|
||||
repeated int64 Timestamps = 6;
|
||||
}
|
||||
|
|
|
|||
104
iterator.go
104
iterator.go
|
|
@ -6,19 +6,19 @@ import (
|
|||
"github.com/pilosa/pilosa/roaring"
|
||||
)
|
||||
|
||||
// Iterator is an interface for looping over bitmap/profile pairs.
|
||||
// Iterator is an interface for looping over row/column pairs.
|
||||
type Iterator interface {
|
||||
Seek(bitmapID, profileID uint64)
|
||||
Next() (bitmapID, profileID uint64, eof bool)
|
||||
Seek(rowID, columnID uint64)
|
||||
Next() (rowID, columnID uint64, eof bool)
|
||||
}
|
||||
|
||||
// BufIterator wraps an iterator to provide the ability to unread values.
|
||||
type BufIterator struct {
|
||||
buf struct {
|
||||
bitmapID uint64
|
||||
profileID uint64
|
||||
eof bool
|
||||
full bool
|
||||
rowID uint64
|
||||
columnID uint64
|
||||
eof bool
|
||||
full bool
|
||||
}
|
||||
itr Iterator
|
||||
}
|
||||
|
|
@ -29,28 +29,28 @@ func NewBufIterator(itr Iterator) *BufIterator {
|
|||
}
|
||||
|
||||
// Seek moves to the first pair equal to or greater than pseek/bseek.
|
||||
func (itr *BufIterator) Seek(bitmapID, profileID uint64) {
|
||||
func (itr *BufIterator) Seek(rowID, columnID uint64) {
|
||||
itr.buf.full = false
|
||||
itr.itr.Seek(bitmapID, profileID)
|
||||
itr.itr.Seek(rowID, columnID)
|
||||
}
|
||||
|
||||
// Next returns the next pair in the bitmap.
|
||||
// Next returns the next pair in the row.
|
||||
// If a value has been buffered then it is returned and the buffer is cleared.
|
||||
func (itr *BufIterator) Next() (bitmapID, profileID uint64, eof bool) {
|
||||
func (itr *BufIterator) Next() (rowID, columnID uint64, eof bool) {
|
||||
if itr.buf.full {
|
||||
itr.buf.full = false
|
||||
return itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof
|
||||
return itr.buf.rowID, itr.buf.columnID, itr.buf.eof
|
||||
}
|
||||
|
||||
// Read values onto buffer in case of unread.
|
||||
itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof = itr.itr.Next()
|
||||
itr.buf.rowID, itr.buf.columnID, itr.buf.eof = itr.itr.Next()
|
||||
|
||||
return itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof
|
||||
return itr.buf.rowID, itr.buf.columnID, itr.buf.eof
|
||||
}
|
||||
|
||||
// Peek reads the next value but leaves it on the buffer.
|
||||
func (itr *BufIterator) Peek() (bitmapID, profileID uint64, eof bool) {
|
||||
bitmapID, profileID, eof = itr.Next()
|
||||
func (itr *BufIterator) Peek() (rowID, columnID uint64, eof bool) {
|
||||
rowID, columnID, eof = itr.Next()
|
||||
itr.Unread()
|
||||
return
|
||||
}
|
||||
|
|
@ -64,30 +64,30 @@ func (itr *BufIterator) Unread() {
|
|||
itr.buf.full = true
|
||||
}
|
||||
|
||||
// LimitIterator wraps an Iterator and limits it to a max profile/bitmap pair.
|
||||
// LimitIterator wraps an Iterator and limits it to a max column/row pair.
|
||||
type LimitIterator struct {
|
||||
itr Iterator
|
||||
maxBitmapID uint64
|
||||
maxProfileID uint64
|
||||
itr Iterator
|
||||
maxRowID uint64
|
||||
maxColumnID uint64
|
||||
|
||||
eof bool
|
||||
}
|
||||
|
||||
// NewLimitIterator returns a new LimitIterator.
|
||||
func NewLimitIterator(itr Iterator, maxBitmapID, maxProfileID uint64) *LimitIterator {
|
||||
func NewLimitIterator(itr Iterator, maxRowID, maxColumnID uint64) *LimitIterator {
|
||||
return &LimitIterator{
|
||||
itr: itr,
|
||||
maxBitmapID: maxBitmapID,
|
||||
maxProfileID: maxProfileID,
|
||||
itr: itr,
|
||||
maxRowID: maxRowID,
|
||||
maxColumnID: maxColumnID,
|
||||
}
|
||||
}
|
||||
|
||||
// Seek moves the underlying iterator to a profile/bitmap pair.
|
||||
func (itr *LimitIterator) Seek(bitmapID, profileID uint64) { itr.itr.Seek(bitmapID, profileID) }
|
||||
// Seek moves the underlying iterator to a column/row pair.
|
||||
func (itr *LimitIterator) Seek(rowID, columnID uint64) { itr.itr.Seek(rowID, columnID) }
|
||||
|
||||
// Next returns the next bitmap/profile ID pair.
|
||||
// Next returns the next row/column ID pair.
|
||||
// If the underlying iterator returns a pair higher than the max then EOF is returned.
|
||||
func (itr *LimitIterator) Next() (bitmapID, profileID uint64, eof bool) {
|
||||
func (itr *LimitIterator) Next() (rowID, columnID uint64, eof bool) {
|
||||
// Always return EOF once it is reached by limit or the underlying iterator.
|
||||
if itr.eof {
|
||||
return 0, 0, true
|
||||
|
|
@ -95,35 +95,35 @@ func (itr *LimitIterator) Next() (bitmapID, profileID uint64, eof bool) {
|
|||
|
||||
// Retrieve pair from underlying iterator.
|
||||
// Mark as EOF if it is beyond the limit (or at EOF).
|
||||
bitmapID, profileID, eof = itr.itr.Next()
|
||||
if eof || bitmapID > itr.maxBitmapID || (bitmapID == itr.maxBitmapID && profileID > itr.maxProfileID) {
|
||||
rowID, columnID, eof = itr.itr.Next()
|
||||
if eof || rowID > itr.maxRowID || (rowID == itr.maxRowID && columnID > itr.maxColumnID) {
|
||||
itr.eof = true
|
||||
return 0, 0, true
|
||||
}
|
||||
|
||||
return bitmapID, profileID, false
|
||||
return rowID, columnID, false
|
||||
}
|
||||
|
||||
// SliceIterator iterates over a pair of bitmap/profile ID slices.
|
||||
// SliceIterator iterates over a pair of row/column ID slices.
|
||||
type SliceIterator struct {
|
||||
bitmapIDs []uint64
|
||||
profileIDs []uint64
|
||||
rowIDs []uint64
|
||||
columnIDs []uint64
|
||||
|
||||
i, n int
|
||||
}
|
||||
|
||||
// NewSliceIterator returns an iterator to iterate over a set of bitmap/profile ID pairs.
|
||||
// NewSliceIterator returns an iterator to iterate over a set of row/column ID pairs.
|
||||
// Both slices MUST have an equal length. Otherwise the function will panic.
|
||||
func NewSliceIterator(bitmapIDs, profileIDs []uint64) *SliceIterator {
|
||||
if len(profileIDs) != len(bitmapIDs) {
|
||||
panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(bitmapIDs), len(profileIDs)))
|
||||
func NewSliceIterator(rowIDs, columnIDs []uint64) *SliceIterator {
|
||||
if len(columnIDs) != len(rowIDs) {
|
||||
panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(rowIDs), len(columnIDs)))
|
||||
}
|
||||
|
||||
return &SliceIterator{
|
||||
bitmapIDs: bitmapIDs,
|
||||
profileIDs: profileIDs,
|
||||
rowIDs: rowIDs,
|
||||
columnIDs: columnIDs,
|
||||
|
||||
n: len(bitmapIDs),
|
||||
n: len(rowIDs),
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -131,10 +131,10 @@ func NewSliceIterator(bitmapIDs, profileIDs []uint64) *SliceIterator {
|
|||
// If the pair is not found, the iterator seeks to the next pair.
|
||||
func (itr *SliceIterator) Seek(bseek, pseek uint64) {
|
||||
for i := 0; i < itr.n; i++ {
|
||||
bitmapID := itr.bitmapIDs[i]
|
||||
profileID := itr.profileIDs[i]
|
||||
rowID := itr.rowIDs[i]
|
||||
columnID := itr.columnIDs[i]
|
||||
|
||||
if (bseek == bitmapID && pseek <= profileID) || bseek < bitmapID {
|
||||
if (bseek == rowID && pseek <= columnID) || bseek < rowID {
|
||||
itr.i = i
|
||||
return
|
||||
}
|
||||
|
|
@ -144,20 +144,20 @@ func (itr *SliceIterator) Seek(bseek, pseek uint64) {
|
|||
itr.i = itr.n
|
||||
}
|
||||
|
||||
// Next returns the next bitmap/profile ID pair.
|
||||
func (itr *SliceIterator) Next() (bitmapID, profileID uint64, eof bool) {
|
||||
// Next returns the next row/column ID pair.
|
||||
func (itr *SliceIterator) Next() (rowID, columnID uint64, eof bool) {
|
||||
if itr.i >= itr.n {
|
||||
return 0, 0, true
|
||||
}
|
||||
|
||||
bitmapID = itr.bitmapIDs[itr.i]
|
||||
profileID = itr.profileIDs[itr.i]
|
||||
rowID = itr.rowIDs[itr.i]
|
||||
columnID = itr.columnIDs[itr.i]
|
||||
|
||||
itr.i++
|
||||
return bitmapID, profileID, false
|
||||
return rowID, columnID, false
|
||||
}
|
||||
|
||||
// RoaringIterator converts a roaring.Iterator to output profile/bitmap pairs.
|
||||
// RoaringIterator converts a roaring.Iterator to output column/row pairs.
|
||||
type RoaringIterator struct {
|
||||
itr *roaring.Iterator
|
||||
}
|
||||
|
|
@ -173,8 +173,8 @@ func (itr *RoaringIterator) Seek(bseek, pseek uint64) {
|
|||
itr.itr.Seek((bseek * SliceWidth) + pseek)
|
||||
}
|
||||
|
||||
// Next returns the next profile/bitmap ID pair.
|
||||
func (itr *RoaringIterator) Next() (bitmapID, profileID uint64, eof bool) {
|
||||
// Next returns the next column/row ID pair.
|
||||
func (itr *RoaringIterator) Next() (rowID, columnID uint64, eof bool) {
|
||||
v, eof := itr.itr.Next()
|
||||
return v / SliceWidth, v % SliceWidth, eof
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,37 +0,0 @@
|
|||
package pilosa_test
|
||||
|
||||
import (
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/gogo/protobuf/proto"
|
||||
"github.com/pilosa/pilosa"
|
||||
"github.com/pilosa/pilosa/internal"
|
||||
)
|
||||
|
||||
// Ensure a message can be marshaled and unmarshaled.
|
||||
func TestMessage_Marshal(t *testing.T) {
|
||||
|
||||
testMessageMarshal(t, &internal.CreateSliceMessage{
|
||||
DB: "d",
|
||||
Slice: 8,
|
||||
})
|
||||
|
||||
testMessageMarshal(t, &internal.DeleteDBMessage{
|
||||
DB: "d",
|
||||
})
|
||||
}
|
||||
|
||||
func testMessageMarshal(t *testing.T, m proto.Message) {
|
||||
marshalled, err := pilosa.MarshalMessage(m)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
unmarshalled, err := pilosa.UnmarshalMessage(marshalled)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !reflect.DeepEqual(unmarshalled, m) {
|
||||
t.Fatalf("unexpected message marshalling: %s", unmarshalled)
|
||||
}
|
||||
}
|
||||
54
pilosa.go
54
pilosa.go
|
|
@ -11,9 +11,9 @@ import (
|
|||
var (
|
||||
ErrHostRequired = errors.New("host required")
|
||||
|
||||
ErrDatabaseRequired = errors.New("database required")
|
||||
ErrDatabaseExists = errors.New("database already exists")
|
||||
ErrDatabaseNotFound = errors.New("database not found")
|
||||
ErrIndexRequired = errors.New("index required")
|
||||
ErrIndexExists = errors.New("index already exists")
|
||||
ErrIndexNotFound = errors.New("index not found")
|
||||
|
||||
// ErrFrameRequired is returned when no frame is specified.
|
||||
ErrFrameRequired = errors.New("frame required")
|
||||
|
|
@ -24,65 +24,65 @@ var (
|
|||
ErrInvalidView = errors.New("invalid view")
|
||||
ErrInvalidCacheType = errors.New("invalid cache type")
|
||||
|
||||
ErrName = errors.New("invalid database or frame's name, must match [a-z0-9_-]")
|
||||
ErrName = errors.New("invalid index or frame's name, must match [a-z0-9_-]")
|
||||
|
||||
// ErrFragmentNotFound is returned when a fragment does not exist.
|
||||
ErrFragmentNotFound = errors.New("fragment not found")
|
||||
ErrQueryRequired = errors.New("query required")
|
||||
)
|
||||
|
||||
// Regular expression to valuate db and frame's name
|
||||
// Regular expression to valuate index and frame's name
|
||||
// Todo: remove . when frame doesn't require . for topN
|
||||
var nameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,64}$`)
|
||||
|
||||
// Profile represents vertical column in a database.
|
||||
// A profile can have a set of attributes attached to it.
|
||||
type Profile struct {
|
||||
// ColumnAttrSet represents a set of attributes for a vertical column in an index.
|
||||
// Can have a set of attributes attached to it.
|
||||
type ColumnAttrSet struct {
|
||||
ID uint64 `json:"id"`
|
||||
Attrs map[string]interface{} `json:"attrs,omitempty"`
|
||||
}
|
||||
|
||||
// encodeProfiles converts a into its internal representation.
|
||||
func encodeProfiles(a []*Profile) []*internal.Profile {
|
||||
other := make([]*internal.Profile, len(a))
|
||||
// encodeColumnAttrSets converts a into its internal representation.
|
||||
func encodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet {
|
||||
other := make([]*internal.ColumnAttrSet, len(a))
|
||||
for i := range a {
|
||||
other[i] = encodeProfile(a[i])
|
||||
other[i] = encodeColumnAttrSet(a[i])
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
||||
// decodeProfiles converts a from its internal representation.
|
||||
func decodeProfiles(a []*internal.Profile) []*Profile {
|
||||
other := make([]*Profile, len(a))
|
||||
// decodeColumnAttrSets converts a from its internal representation.
|
||||
func decodeColumnAttrSets(a []*internal.ColumnAttrSet) []*ColumnAttrSet {
|
||||
other := make([]*ColumnAttrSet, len(a))
|
||||
for i := range a {
|
||||
other[i] = decodeProfile(a[i])
|
||||
other[i] = decodeColumnAttrSet(a[i])
|
||||
}
|
||||
return other
|
||||
}
|
||||
|
||||
// encodeProfile converts p into its internal representation.
|
||||
func encodeProfile(p *Profile) *internal.Profile {
|
||||
return &internal.Profile{
|
||||
ID: p.ID,
|
||||
Attrs: encodeAttrs(p.Attrs),
|
||||
// encodeColumnAttrSet converts set into its internal representation.
|
||||
func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
|
||||
return &internal.ColumnAttrSet{
|
||||
ID: set.ID,
|
||||
Attrs: encodeAttrs(set.Attrs),
|
||||
}
|
||||
}
|
||||
|
||||
// decodeProfile converts b from its internal representation.
|
||||
func decodeProfile(pb *internal.Profile) *Profile {
|
||||
p := &Profile{
|
||||
// decodeColumnAttrSet converts b from its internal representation.
|
||||
func decodeColumnAttrSet(pb *internal.ColumnAttrSet) *ColumnAttrSet {
|
||||
set := &ColumnAttrSet{
|
||||
ID: pb.ID,
|
||||
}
|
||||
|
||||
if len(pb.Attrs) > 0 {
|
||||
p.Attrs = make(map[string]interface{}, len(pb.Attrs))
|
||||
set.Attrs = make(map[string]interface{}, len(pb.Attrs))
|
||||
for _, attr := range pb.Attrs {
|
||||
k, v := decodeAttr(attr)
|
||||
p.Attrs[k] = v
|
||||
set.Attrs[k] = v
|
||||
}
|
||||
}
|
||||
|
||||
return p
|
||||
return set
|
||||
}
|
||||
|
||||
// TimeFormat is the go-style time format used to parse string dates.
|
||||
|
|
|
|||
192
server.go
192
server.go
|
|
@ -26,7 +26,7 @@ const (
|
|||
DefaultPollingInterval = 60 * time.Second
|
||||
)
|
||||
|
||||
// Server represents an index wrapped by a running HTTP server.
|
||||
// Server represents a holder wrapped by a running HTTP server.
|
||||
type Server struct {
|
||||
ln net.Listener
|
||||
|
||||
|
|
@ -35,7 +35,7 @@ type Server struct {
|
|||
closing chan struct{}
|
||||
|
||||
// Data storage and HTTP interface.
|
||||
Index *Index
|
||||
Holder *Holder
|
||||
Handler *Handler
|
||||
Broadcaster Broadcaster
|
||||
BroadcastReceiver BroadcastReceiver
|
||||
|
|
@ -61,7 +61,7 @@ func NewServer() *Server {
|
|||
s := &Server{
|
||||
closing: make(chan struct{}),
|
||||
|
||||
Index: NewIndex(),
|
||||
Holder: NewHolder(),
|
||||
Handler: NewHandler(),
|
||||
Broadcaster: NopBroadcaster,
|
||||
BroadcastReceiver: NopBroadcastReceiver,
|
||||
|
|
@ -73,7 +73,8 @@ func NewServer() *Server {
|
|||
LogOutput: os.Stderr,
|
||||
}
|
||||
|
||||
s.Handler.Index = s.Index
|
||||
s.Handler.Holder = s.Holder
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
|
|
@ -102,8 +103,8 @@ func (s *Server) Open() error {
|
|||
s.Cluster.Nodes = []*Node{{Host: s.Host}}
|
||||
}
|
||||
|
||||
// Open index.
|
||||
if err := s.Index.Open(); err != nil {
|
||||
// Open holder.
|
||||
if err := s.Holder.Open(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
|
@ -118,20 +119,21 @@ func (s *Server) Open() error {
|
|||
|
||||
// Create executor for executing queries.
|
||||
e := NewExecutor()
|
||||
e.Index = s.Index
|
||||
e.Holder = s.Holder
|
||||
e.Host = s.Host
|
||||
e.Cluster = s.Cluster
|
||||
|
||||
// Initialize HTTP handler.
|
||||
s.Handler.Broadcaster = s.Broadcaster
|
||||
s.Handler.StatusHandler = s
|
||||
s.Handler.Host = s.Host
|
||||
s.Handler.Cluster = s.Cluster
|
||||
s.Handler.Executor = e
|
||||
s.Handler.LogOutput = s.LogOutput
|
||||
|
||||
// Initialize Index.
|
||||
s.Index.Broadcaster = s.Broadcaster
|
||||
s.Index.LogOutput = s.LogOutput
|
||||
// Initialize Holder.
|
||||
s.Holder.Broadcaster = s.Broadcaster
|
||||
s.Holder.LogOutput = s.LogOutput
|
||||
|
||||
// Serve HTTP.
|
||||
go func() { http.Serve(ln, s.Handler) }()
|
||||
|
|
@ -154,8 +156,8 @@ func (s *Server) Close() error {
|
|||
if s.ln != nil {
|
||||
s.ln.Close()
|
||||
}
|
||||
if s.Index != nil {
|
||||
s.Index.Close()
|
||||
if s.Holder != nil {
|
||||
s.Holder.Close()
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
@ -172,10 +174,11 @@ func (s *Server) Addr() net.Addr {
|
|||
func (s *Server) logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) }
|
||||
|
||||
func (s *Server) monitorAntiEntropy() {
|
||||
t := time.Now()
|
||||
ticker := time.NewTicker(s.AntiEntropyInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
s.logger().Printf("index sync monitor initializing (%s interval)", s.AntiEntropyInterval)
|
||||
s.logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval)
|
||||
|
||||
for {
|
||||
// Wait for tick or a close.
|
||||
|
|
@ -183,26 +186,30 @@ func (s *Server) monitorAntiEntropy() {
|
|||
case <-s.closing:
|
||||
return
|
||||
case <-ticker.C:
|
||||
s.Holder.Stats.Count("AntiEntropy", 1)
|
||||
}
|
||||
|
||||
s.logger().Printf("index sync beginning")
|
||||
s.logger().Printf("holder sync beginning")
|
||||
|
||||
// Initialize syncer with local index and remote client.
|
||||
var syncer IndexSyncer
|
||||
syncer.Index = s.Index
|
||||
// Initialize syncer with local holder and remote client.
|
||||
var syncer HolderSyncer
|
||||
syncer.Holder = s.Holder
|
||||
syncer.Host = s.Host
|
||||
syncer.Cluster = s.Cluster
|
||||
syncer.Closing = s.closing
|
||||
|
||||
// Sync indexes.
|
||||
if err := syncer.SyncIndex(); err != nil {
|
||||
s.logger().Printf("index sync error: err=%s", err)
|
||||
// Sync holders.
|
||||
if err := syncer.SyncHolder(); err != nil {
|
||||
s.logger().Printf("holder sync error: err=%s", err)
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
// Record successful sync in log.
|
||||
s.logger().Printf("index sync complete")
|
||||
s.logger().Printf("holder sync complete")
|
||||
}
|
||||
dif := time.Since(t)
|
||||
s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif))
|
||||
}
|
||||
|
||||
// monitorMaxSlices periodically pulls the highest slice from each node in the cluster.
|
||||
|
|
@ -222,20 +229,20 @@ func (s *Server) monitorMaxSlices() {
|
|||
case <-ticker.C:
|
||||
}
|
||||
|
||||
oldmaxslices := s.Index.MaxSlices()
|
||||
oldmaxslices := s.Holder.MaxSlices()
|
||||
for _, node := range s.Cluster.Nodes {
|
||||
if s.Host != node.Host {
|
||||
maxSlices, _ := checkMaxSlices(node.Host)
|
||||
for db, newmax := range maxSlices {
|
||||
// if we don't know about a db locally, log an error because
|
||||
// db's should be created and synced prior to slice creation
|
||||
if localdb := s.Index.DB(db); localdb != nil {
|
||||
if newmax > oldmaxslices[db] {
|
||||
oldmaxslices[db] = newmax
|
||||
localdb.SetRemoteMaxSlice(newmax)
|
||||
for index, newmax := range maxSlices {
|
||||
// if we don't know about an index locally, log an error because
|
||||
// indexes should be created and synced prior to slice creation
|
||||
if localIndex := s.Holder.Index(index); localIndex != nil {
|
||||
if newmax > oldmaxslices[index] {
|
||||
oldmaxslices[index] = newmax
|
||||
localIndex.SetRemoteMaxSlice(newmax)
|
||||
}
|
||||
} else {
|
||||
s.logger().Printf("Local DB not found: %s", db)
|
||||
s.logger().Printf("Local Index not found: %s", index)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -243,81 +250,103 @@ func (s *Server) monitorMaxSlices() {
|
|||
}
|
||||
}
|
||||
|
||||
// LocalState returns the state of the local node as well as the
|
||||
// index (dbs/frames) according to the local node.
|
||||
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
|
||||
func (s *Server) LocalState() (proto.Message, error) {
|
||||
if s.Index == nil {
|
||||
return nil, errors.New("Server.Index is nil.")
|
||||
}
|
||||
return &internal.NodeState{
|
||||
Host: s.Host,
|
||||
State: "OK", // TODO: make this work, pull from s.Cluster.Node
|
||||
DBs: encodeDBs(s.Index.DBs()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// ReceiveMessage represents an implementation of BroadcastHandler.
|
||||
func (s *Server) ReceiveMessage(pb proto.Message) error {
|
||||
switch obj := pb.(type) {
|
||||
case *internal.CreateSliceMessage:
|
||||
d := s.Index.DB(obj.DB)
|
||||
if d == nil {
|
||||
return fmt.Errorf("Local DB not found: %s", obj.DB)
|
||||
idx := s.Holder.Index(obj.Index)
|
||||
if idx == nil {
|
||||
return fmt.Errorf("Local Index not found: %s", obj.Index)
|
||||
}
|
||||
d.SetRemoteMaxSlice(obj.Slice)
|
||||
case *internal.CreateDBMessage:
|
||||
opt := DBOptions{ColumnLabel: obj.Meta.ColumnLabel}
|
||||
_, err := s.Index.CreateDB(obj.DB, opt)
|
||||
idx.SetRemoteMaxSlice(obj.Slice)
|
||||
case *internal.CreateIndexMessage:
|
||||
opt := IndexOptions{ColumnLabel: obj.Meta.ColumnLabel}
|
||||
_, err := s.Holder.CreateIndex(obj.Index, opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case *internal.DeleteDBMessage:
|
||||
fmt.Println("DELETE:", obj.DB)
|
||||
if err := s.Index.DeleteDB(obj.DB); err != nil {
|
||||
case *internal.DeleteIndexMessage:
|
||||
if err := s.Holder.DeleteIndex(obj.Index); err != nil {
|
||||
return err
|
||||
}
|
||||
case *internal.CreateFrameMessage:
|
||||
db := s.Index.DB(obj.DB)
|
||||
index := s.Holder.Index(obj.Index)
|
||||
opt := FrameOptions{RowLabel: obj.Meta.RowLabel}
|
||||
_, err := db.CreateFrame(obj.Frame, opt)
|
||||
_, err := index.CreateFrame(obj.Frame, opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
case *internal.DeleteFrameMessage:
|
||||
db := s.Index.DB(obj.DB)
|
||||
if err := db.DeleteFrame(obj.Frame); err != nil {
|
||||
index := s.Holder.Index(obj.Index)
|
||||
if err := index.DeleteFrame(obj.Frame); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// HandleRemoteState receives incoming NodeState from remote nodes.
|
||||
func (s *Server) HandleRemoteState(pb proto.Message) error {
|
||||
return s.mergeRemoteState(pb.(*internal.NodeState))
|
||||
// Server implements StatusHandler.
|
||||
// LocalStatus returns the state of the local node as well as the
|
||||
// holder (indexes/frames) according to the local node.
|
||||
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
|
||||
func (s *Server) LocalStatus() (proto.Message, error) {
|
||||
if s.Holder == nil {
|
||||
return nil, errors.New("Server.Holder is nil")
|
||||
}
|
||||
return &internal.NodeStatus{
|
||||
Host: s.Host,
|
||||
State: NodeStateUp,
|
||||
Indexes: encodeIndexes(s.Holder.Indexes()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Server) mergeRemoteState(ns *internal.NodeState) error {
|
||||
// TODO: update some node state value in the cluster (it should be in cluster.node i guess)
|
||||
// ClusterStatus returns the NodeState for all nodes in the cluster.
|
||||
func (s *Server) ClusterStatus() (proto.Message, error) {
|
||||
// Update local Node.state.
|
||||
ns, err := s.LocalStatus()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
node := s.Cluster.NodeByHost(s.Host)
|
||||
node.SetStatus(ns.(*internal.NodeStatus))
|
||||
|
||||
// Create databases that don't exist.
|
||||
for _, db := range ns.DBs {
|
||||
opt := DBOptions{
|
||||
ColumnLabel: db.Meta.ColumnLabel,
|
||||
TimeQuantum: TimeQuantum(db.Meta.TimeQuantum),
|
||||
// Update NodeState for all nodes.
|
||||
for host, nodeState := range s.Cluster.NodeStates() {
|
||||
node := s.Cluster.NodeByHost(host)
|
||||
node.SetState(nodeState)
|
||||
}
|
||||
|
||||
return s.Cluster.Status(), nil
|
||||
}
|
||||
|
||||
// HandleRemoteStatus receives incoming NodeState from remote nodes.
|
||||
func (s *Server) HandleRemoteStatus(pb proto.Message) error {
|
||||
return s.mergeRemoteStatus(pb.(*internal.NodeStatus))
|
||||
}
|
||||
|
||||
func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
|
||||
// Update Node.state.
|
||||
node := s.Cluster.NodeByHost(ns.Host)
|
||||
node.SetStatus(ns)
|
||||
|
||||
// Create indexes that don't exist.
|
||||
for _, index := range ns.Indexes {
|
||||
opt := IndexOptions{
|
||||
ColumnLabel: index.Meta.ColumnLabel,
|
||||
TimeQuantum: TimeQuantum(index.Meta.TimeQuantum),
|
||||
}
|
||||
d, err := s.Index.CreateDBIfNotExists(db.Name, opt)
|
||||
idx, err := s.Holder.CreateIndexIfNotExists(index.Name, opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Create frames that don't exist.
|
||||
for _, f := range db.Frames {
|
||||
for _, f := range index.Frames {
|
||||
opt := FrameOptions{
|
||||
RowLabel: f.Meta.RowLabel,
|
||||
TimeQuantum: TimeQuantum(f.Meta.TimeQuantum),
|
||||
CacheSize: f.Meta.CacheSize,
|
||||
}
|
||||
_, err := d.CreateFrameIfNotExists(f.Name, opt)
|
||||
_, err := idx.CreateFrameIfNotExists(f.Name, opt)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -390,18 +419,21 @@ func (s *Server) monitorRuntime() {
|
|||
return
|
||||
case <-gcn.AfterGC():
|
||||
// GC just ran
|
||||
s.Index.Stats.Count("garbage_collection", 1)
|
||||
s.logger().Printf("garbage collection complete")
|
||||
s.Holder.Stats.Count("garbage_collection", 1)
|
||||
case <-ticker.C:
|
||||
}
|
||||
|
||||
s.logger().Printf("runtime stats beginning")
|
||||
|
||||
// TODO
|
||||
s.Index.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()))
|
||||
|
||||
// Record successful sync in log.
|
||||
s.logger().Printf("runtime stats complete")
|
||||
// Record the number of go routines
|
||||
s.Holder.Stats.Gauge("goroutines", float64(runtime.NumGoroutine()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// StatusHandler specifies two methods which an object must implement to share
|
||||
// state in the cluster. These are used by the GossipNodeSet to implement the
|
||||
// LocalState and MergeRemoteState methods of memberlist.Delegate
|
||||
type StatusHandler interface {
|
||||
LocalStatus() (proto.Message, error)
|
||||
ClusterStatus() (proto.Message, error)
|
||||
HandleRemoteStatus(proto.Message) error
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
// package server contains the `pilosa server` subcommand which runs Pilosa
|
||||
// Package server contains the `pilosa server` subcommand which runs Pilosa
|
||||
// itself. The purpose of this package is to define an easily tested Command
|
||||
// object which handles interpreting configuration and setting up all the
|
||||
// objects that Pilosa needs.
|
||||
|
|
@ -51,7 +51,7 @@ type Command struct {
|
|||
Done chan struct{}
|
||||
}
|
||||
|
||||
// NewMain returns a new instance of Main.
|
||||
// NewCommand returns a new instance of Main.
|
||||
func NewCommand(stdin io.Reader, stdout, stderr io.Writer) *Command {
|
||||
return &Command{
|
||||
Server: pilosa.NewServer(),
|
||||
|
|
@ -90,6 +90,7 @@ func (m *Command) Run(args ...string) (err error) {
|
|||
return nil
|
||||
}
|
||||
|
||||
// SetupServer use the cluster configuration to setup this server
|
||||
func (m *Command) SetupServer() error {
|
||||
var err error
|
||||
cluster := pilosa.NewCluster()
|
||||
|
|
@ -117,11 +118,11 @@ func (m *Command) SetupServer() error {
|
|||
m.Server.LogOutput = logFile
|
||||
}
|
||||
|
||||
// Configure index.
|
||||
// Configure holder.
|
||||
fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir)
|
||||
m.Server.Index.Path = m.Config.DataDir
|
||||
m.Server.Holder.Path = m.Config.DataDir
|
||||
m.Server.MetricInterval = time.Duration(m.Config.Metric.PollingInterval)
|
||||
m.Server.Index.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
|
||||
m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -168,8 +169,12 @@ func (m *Command) SetupServer() error {
|
|||
m.Server.Broadcaster = pilosa.NopBroadcaster
|
||||
m.Server.Cluster.NodeSet = pilosa.NewStaticNodeSet()
|
||||
m.Server.BroadcastReceiver = pilosa.NopBroadcastReceiver
|
||||
err := m.Server.Cluster.NodeSet.(*pilosa.StaticNodeSet).Join(m.Server.Cluster.Nodes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("'%v' is not a supported value for broadcaster type.", m.Config.Cluster.Type)
|
||||
return fmt.Errorf("'%v' is not a supported value for broadcaster type", m.Config.Cluster.Type)
|
||||
}
|
||||
|
||||
// Set configuration options.
|
||||
|
|
|
|||
|
|
@ -39,29 +39,29 @@ func TestMain_Set_Quick(t *testing.T) {
|
|||
|
||||
// Execute SetBit() commands.
|
||||
for _, cmd := range cmds {
|
||||
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
|
||||
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := client.CreateFrame(context.Background(), "d", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists {
|
||||
if err := client.CreateFrame(context.Background(), "i", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := m.Query("d", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil {
|
||||
if _, err := m.Query("i", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate data.
|
||||
for frame, frameSet := range SetCommands(cmds).Frames() {
|
||||
for id, profileIDs := range frameSet {
|
||||
for id, columnIDs := range frameSet {
|
||||
exp := MustMarshalJSON(map[string]interface{}{
|
||||
"results": []interface{}{
|
||||
map[string]interface{}{
|
||||
"bits": profileIDs,
|
||||
"bits": columnIDs,
|
||||
"attrs": map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
}) + "\n"
|
||||
if res, err := m.Query("d", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
|
||||
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != exp {
|
||||
t.Fatalf("unexpected result:\n\ngot=%s\n\nexp=%s\n\n", res, exp)
|
||||
|
|
@ -75,16 +75,16 @@ func TestMain_Set_Quick(t *testing.T) {
|
|||
|
||||
// Validate data after reopening.
|
||||
for frame, frameSet := range SetCommands(cmds).Frames() {
|
||||
for id, profileIDs := range frameSet {
|
||||
for id, columnIDs := range frameSet {
|
||||
exp := MustMarshalJSON(map[string]interface{}{
|
||||
"results": []interface{}{
|
||||
map[string]interface{}{
|
||||
"bits": profileIDs,
|
||||
"bits": columnIDs,
|
||||
"attrs": map[string]interface{}{},
|
||||
},
|
||||
},
|
||||
}) + "\n"
|
||||
if res, err := m.Query("d", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
|
||||
if res, err := m.Query("i", "", fmt.Sprintf(`Bitmap(id=%d, frame=%q)`, id, frame)); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != exp {
|
||||
t.Fatalf("unexpected result (reopen):\n\ngot=%s\n\nexp=%s\n\n", res, exp)
|
||||
|
|
@ -102,54 +102,54 @@ func TestMain_Set_Quick(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// Ensure program can set bitmap attributes and retrieve them.
|
||||
func TestMain_SetBitmapAttrs(t *testing.T) {
|
||||
// Ensure program can set row attributes and retrieve them.
|
||||
func TestMain_SetRowAttrs(t *testing.T) {
|
||||
m := MustRunMain()
|
||||
defer m.Close()
|
||||
|
||||
// Create frames.
|
||||
client := m.Client()
|
||||
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
|
||||
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
|
||||
} else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := client.CreateFrame(context.Background(), "d", "z", pilosa.FrameOptions{}); err != nil {
|
||||
} else if err := client.CreateFrame(context.Background(), "i", "z", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := client.CreateFrame(context.Background(), "d", "neg", pilosa.FrameOptions{}); err != nil {
|
||||
} else if err := client.CreateFrame(context.Background(), "i", "neg", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bits on different bitmaps in different frames.
|
||||
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil {
|
||||
// Set bits on different rows in different frames.
|
||||
if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := m.Query("d", "", `SetBit(id=2, frame="x.n", profileID=100)`); err != nil {
|
||||
} else if _, err := m.Query("i", "", `SetBit(id=2, frame="x.n", columnID=100)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := m.Query("d", "", `SetBit(id=2, frame="z", profileID=100)`); err != nil {
|
||||
} else if _, err := m.Query("i", "", `SetBit(id=2, frame="z", columnID=100)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := m.Query("d", "", `SetBit(id=3, frame="neg", profileID=100)`); err != nil {
|
||||
} else if _, err := m.Query("i", "", `SetBit(id=3, frame="neg", columnID=100)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bitmap attributes.
|
||||
if _, err := m.Query("d", "", `SetBitmapAttrs(id=1, frame="x.n", x=100)`); err != nil {
|
||||
// Set row attributes.
|
||||
if _, err := m.Query("i", "", `SetRowAttrs(id=1, frame="x.n", x=100)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := m.Query("d", "", `SetBitmapAttrs(id=2, frame="x.n", x=-200)`); err != nil {
|
||||
} else if _, err := m.Query("i", "", `SetRowAttrs(id=2, frame="x.n", x=-200)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := m.Query("d", "", `SetBitmapAttrs(id=2, frame="z", x=300)`); err != nil {
|
||||
} else if _, err := m.Query("i", "", `SetRowAttrs(id=2, frame="z", x=300)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := m.Query("d", "", `SetBitmapAttrs(id=3, frame="neg", x=-0.44)`); err != nil {
|
||||
} else if _, err := m.Query("i", "", `SetRowAttrs(id=3, frame="neg", x=-0.44)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Query bitmap x.n/1.
|
||||
if res, err := m.Query("d", "", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
// Query row x.n/1.
|
||||
if res, err := m.Query("i", "", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
|
||||
// Query bitmap x.n/2.
|
||||
if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil {
|
||||
// Query row x.n/2.
|
||||
if res, err := m.Query("i", "", `Bitmap(id=2, frame="x.n")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
|
|
@ -159,55 +159,55 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Query bitmaps after reopening.
|
||||
if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
// Query rows after reopening.
|
||||
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result(reopen): %s", res)
|
||||
}
|
||||
|
||||
if res, err := m.Query("d", "profiles=true", `Bitmap(id=3, frame="neg")`); err != nil {
|
||||
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=3, frame="neg")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result(reopen): %s", res)
|
||||
}
|
||||
// Query bitmap x.n/2.
|
||||
if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil {
|
||||
// Query row x.n/2.
|
||||
if res, err := m.Query("i", "", `Bitmap(id=2, frame="x.n")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure program can set profile attributes and retrieve them.
|
||||
func TestMain_SetProfileAttrs(t *testing.T) {
|
||||
// Ensure program can set column attributes and retrieve them.
|
||||
func TestMain_SetColumnAttrs(t *testing.T) {
|
||||
m := MustRunMain()
|
||||
defer m.Close()
|
||||
|
||||
// Create frames.
|
||||
client := m.Client()
|
||||
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
|
||||
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
|
||||
} else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bits on bitmap.
|
||||
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil {
|
||||
// Set bits on row.
|
||||
if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=101)`); err != nil {
|
||||
} else if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", columnID=101)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set profile attributes.
|
||||
if _, err := m.Query("d", "", `SetProfileAttrs(id=100, foo="bar")`); err != nil {
|
||||
// Set column attributes.
|
||||
if _, err := m.Query("i", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Query bitmap.
|
||||
if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
// Query row.
|
||||
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
|
||||
|
|
@ -215,43 +215,43 @@ func TestMain_SetProfileAttrs(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Query bitmap after reopening.
|
||||
if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
// Query row after reopening.
|
||||
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
|
||||
t.Fatalf("unexpected result(reopen): %s", res)
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure program can set profile attributes with columnLabel option.
|
||||
func TestMain_SetProfileAttrsWithColumnOption(t *testing.T) {
|
||||
// Ensure program can set column attributes with columnLabel option.
|
||||
func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) {
|
||||
m := MustRunMain()
|
||||
defer m.Close()
|
||||
|
||||
// Create frames.
|
||||
client := m.Client()
|
||||
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrDatabaseExists {
|
||||
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{ColumnLabel: "col"}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := client.CreateFrame(context.Background(), "d", "x.n", pilosa.FrameOptions{}); err != nil {
|
||||
} else if err := client.CreateFrame(context.Background(), "i", "x.n", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set bits on bitmap.
|
||||
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil {
|
||||
// Set bits on row.
|
||||
if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil {
|
||||
} else if _, err := m.Query("i", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Set profile attributes.
|
||||
if _, err := m.Query("d", "", `SetProfileAttrs(col=100, foo="bar")`); err != nil {
|
||||
// Set column attributes.
|
||||
if _, err := m.Query("i", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Query bitmap.
|
||||
if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
// Query row.
|
||||
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
}
|
||||
|
||||
|
|
@ -274,27 +274,27 @@ func TestMain_FrameRestore(t *testing.T) {
|
|||
|
||||
// Create frames.
|
||||
client := m0.Client()
|
||||
if err := client.CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
|
||||
if err := client.CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := client.CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil {
|
||||
} else if err := client.CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Write data on first cluster.
|
||||
if _, err := m0.Query("d", "", `
|
||||
SetBit(id=1, frame="f", profileID=100)
|
||||
SetBit(id=1, frame="f", profileID=1000)
|
||||
SetBit(id=1, frame="f", profileID=100000)
|
||||
SetBit(id=1, frame="f", profileID=200000)
|
||||
SetBit(id=1, frame="f", profileID=400000)
|
||||
SetBit(id=1, frame="f", profileID=600000)
|
||||
SetBit(id=1, frame="f", profileID=800000)
|
||||
if _, err := m0.Query("x", "", `
|
||||
SetBit(id=1, frame="f", columnID=100)
|
||||
SetBit(id=1, frame="f", columnID=1000)
|
||||
SetBit(id=1, frame="f", columnID=100000)
|
||||
SetBit(id=1, frame="f", columnID=200000)
|
||||
SetBit(id=1, frame="f", columnID=400000)
|
||||
SetBit(id=1, frame="f", columnID=600000)
|
||||
SetBit(id=1, frame="f", columnID=800000)
|
||||
`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Query bitmap on first cluster.
|
||||
if res, err := m0.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil {
|
||||
// Query row on first cluster.
|
||||
if res, err := m0.Query("x", "", `Bitmap(id=1, frame="f")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
|
|
@ -308,16 +308,16 @@ func TestMain_FrameRestore(t *testing.T) {
|
|||
client, err := pilosa.NewClient(m2.Server.Host)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := m2.Client().CreateDB(context.Background(), "d", pilosa.DBOptions{}); err != nil && err != pilosa.ErrDatabaseExists {
|
||||
} else if err := m2.Client().CreateIndex(context.Background(), "x", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
|
||||
t.Fatal(err)
|
||||
} else if err := m2.Client().CreateFrame(context.Background(), "d", "f", pilosa.FrameOptions{}); err != nil {
|
||||
} else if err := m2.Client().CreateFrame(context.Background(), "x", "f", pilosa.FrameOptions{}); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "d", "f"); err != nil {
|
||||
} else if err := client.RestoreFrame(context.Background(), m0.Server.Host, "x", "f"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// Query bitmap on second cluster.
|
||||
if res, err := m2.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil {
|
||||
// Query row on second cluster.
|
||||
if res, err := m2.Query("x", "", `Bitmap(id=1, frame="f")`); err != nil {
|
||||
t.Fatal(err)
|
||||
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
|
||||
t.Fatalf("unexpected result: %s", res)
|
||||
|
|
@ -431,8 +431,9 @@ func (m *Main) Client() *pilosa.Client {
|
|||
}
|
||||
|
||||
// Query executes a query against the program through the HTTP API.
|
||||
func (m *Main) Query(db, rawQuery, query string) (string, error) {
|
||||
resp := MustDo("POST", m.URL()+fmt.Sprintf("/db/%s/query?", db)+rawQuery, query)
|
||||
func (m *Main) Query(index, rawQuery, query string) (string, error) {
|
||||
fmt.Println("Query:", index, query)
|
||||
resp := MustDo("POST", m.URL()+fmt.Sprintf("/index/%s/query?", index)+rawQuery, query)
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return "", fmt.Errorf("invalid status: %d, body=%s", resp.StatusCode, resp.Body)
|
||||
}
|
||||
|
|
@ -441,14 +442,14 @@ func (m *Main) Query(db, rawQuery, query string) (string, error) {
|
|||
|
||||
// SetCommand represents a command to set a bit.
|
||||
type SetCommand struct {
|
||||
ID uint64
|
||||
Frame string
|
||||
ProfileID uint64
|
||||
ID uint64
|
||||
Frame string
|
||||
ColumnID uint64
|
||||
}
|
||||
|
||||
type SetCommands []SetCommand
|
||||
|
||||
// Frames returns the set of profile ids for each frame/bitmap.
|
||||
// Frames returns the set of column ids for each frame/row.
|
||||
func (a SetCommands) Frames() map[string]map[uint64][]uint64 {
|
||||
// Create a set of unique commands.
|
||||
m := make(map[SetCommand]struct{})
|
||||
|
|
@ -456,16 +457,16 @@ func (a SetCommands) Frames() map[string]map[uint64][]uint64 {
|
|||
m[cmd] = struct{}{}
|
||||
}
|
||||
|
||||
// Build unique ids for each frame & bitmap.
|
||||
// Build unique ids for each frame & row.
|
||||
frames := make(map[string]map[uint64][]uint64)
|
||||
for cmd := range m {
|
||||
if frames[cmd.Frame] == nil {
|
||||
frames[cmd.Frame] = make(map[uint64][]uint64)
|
||||
}
|
||||
frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ProfileID)
|
||||
frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ColumnID)
|
||||
}
|
||||
|
||||
// Sort each set of profile ids.
|
||||
// Sort each set of column ids.
|
||||
for _, frame := range frames {
|
||||
for id := range frame {
|
||||
sort.Sort(uint64Slice(frame[id]))
|
||||
|
|
@ -480,9 +481,9 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand {
|
|||
cmds := make([]SetCommand, rand.Intn(n))
|
||||
for i := range cmds {
|
||||
cmds[i] = SetCommand{
|
||||
ID: uint64(rand.Intn(1000)),
|
||||
Frame: "x.n",
|
||||
ProfileID: uint64(rand.Intn(10)),
|
||||
ID: uint64(rand.Intn(1000)),
|
||||
Frame: "x.n",
|
||||
ColumnID: uint64(rand.Intn(10)),
|
||||
}
|
||||
}
|
||||
return cmds
|
||||
|
|
|
|||
28
stats.go
28
stats.go
|
|
@ -26,6 +26,9 @@ type StatsClient interface {
|
|||
// Tracks the number of times something occurs per second.
|
||||
Count(name string, value int64)
|
||||
|
||||
// Tracks the number of times something occurs per second with custom tags
|
||||
CountWithCustomTags(name string, value int64, tags []string)
|
||||
|
||||
// Sets the value of a metric.
|
||||
Gauge(name string, value float64)
|
||||
|
||||
|
|
@ -44,13 +47,14 @@ var NopStatsClient StatsClient
|
|||
// nopStatsClient represents a client that doesn't do anything.
|
||||
type nopStatsClient struct{}
|
||||
|
||||
func (c *nopStatsClient) Tags() []string { return nil }
|
||||
func (c *nopStatsClient) WithTags(tags ...string) StatsClient { return c }
|
||||
func (c *nopStatsClient) Count(name string, value int64) {}
|
||||
func (c *nopStatsClient) Gauge(name string, value float64) {}
|
||||
func (c *nopStatsClient) Histogram(name string, value float64) {}
|
||||
func (c *nopStatsClient) Set(name string, value string) {}
|
||||
func (c *nopStatsClient) Timing(name string, value time.Duration) {}
|
||||
func (c *nopStatsClient) Tags() []string { return nil }
|
||||
func (c *nopStatsClient) WithTags(tags ...string) StatsClient { return c }
|
||||
func (c *nopStatsClient) Count(name string, value int64) {}
|
||||
func (c *nopStatsClient) CountWithCustomTags(name string, value int64, tags []string) {}
|
||||
func (c *nopStatsClient) Gauge(name string, value float64) {}
|
||||
func (c *nopStatsClient) Histogram(name string, value float64) {}
|
||||
func (c *nopStatsClient) Set(name string, value string) {}
|
||||
func (c *nopStatsClient) Timing(name string, value time.Duration) {}
|
||||
|
||||
// ExpvarStatsClient writes stats out to expvars.
|
||||
type ExpvarStatsClient struct {
|
||||
|
|
@ -89,6 +93,10 @@ func (c *ExpvarStatsClient) Count(name string, value int64) {
|
|||
c.m.Add(name, value)
|
||||
}
|
||||
|
||||
func (c *ExpvarStatsClient) CountWithCustomTags(name string, value int64, tags []string) {
|
||||
c.m.Add(name, value)
|
||||
}
|
||||
|
||||
// Gauge sets the value of a metric.
|
||||
func (c *ExpvarStatsClient) Gauge(name string, value float64) {
|
||||
var f expvar.Float
|
||||
|
|
@ -142,6 +150,12 @@ func (a MultiStatsClient) Count(name string, value int64) {
|
|||
}
|
||||
}
|
||||
|
||||
func (a MultiStatsClient) CountWithCustomTags(name string, value int64, tags []string) {
|
||||
for _, c := range a {
|
||||
c.CountWithCustomTags(name, value, tags)
|
||||
}
|
||||
}
|
||||
|
||||
// Gauge sets the value of a metric on all clients.
|
||||
func (a MultiStatsClient) Gauge(name string, value float64) {
|
||||
for _, c := range a {
|
||||
|
|
|
|||
164
stats_test.go
Normal file
164
stats_test.go
Normal file
|
|
@ -0,0 +1,164 @@
|
|||
package pilosa_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/pilosa/pilosa"
|
||||
)
|
||||
|
||||
func TestStatsCount_TopN(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth+2)
|
||||
|
||||
// Execute query.
|
||||
called := false
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
e.Holder.Stats = &MockStats{
|
||||
mockCountWithTags: func(name string, value int64, tags []string) {
|
||||
if name != "TopN" {
|
||||
t.Errorf("Expected TopN, Results %s", name)
|
||||
}
|
||||
|
||||
if tags[0] != "index:d" {
|
||||
t.Errorf("Expected db, Results %s", tags[0])
|
||||
}
|
||||
|
||||
called = true
|
||||
return
|
||||
},
|
||||
}
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !called {
|
||||
t.Error("CountWithCustomTags name isn't called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsCount_Bitmap(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
|
||||
called := false
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
e.Holder.Stats = &MockStats{
|
||||
mockCountWithTags: func(name string, value int64, tags []string) {
|
||||
if name != "Bitmap" {
|
||||
t.Errorf("Expected Bitmap, Results %s", name)
|
||||
}
|
||||
|
||||
if tags[0] != "index:d" {
|
||||
t.Errorf("Expected db, Results %s", tags[0])
|
||||
}
|
||||
|
||||
called = true
|
||||
return
|
||||
},
|
||||
}
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(frame=f, id=0)`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !called {
|
||||
t.Error("CountWithCustomTags name isn't called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsCount_SetBitmapAttrs(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1)
|
||||
|
||||
called := false
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
frame := e.Holder.Frame("d", "f")
|
||||
if frame == nil {
|
||||
t.Fatal("frame not found")
|
||||
}
|
||||
|
||||
frame.Stats = &MockStats{
|
||||
mockCount: func(name string, value int64) {
|
||||
if name != "SetBitmapAttrs" {
|
||||
t.Errorf("Expected SetBitmapAttrs, Results %s", name)
|
||||
}
|
||||
called = true
|
||||
return
|
||||
},
|
||||
}
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !called {
|
||||
t.Error("Count isn't called")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatsCount_SetProfileAttrs(t *testing.T) {
|
||||
hldr := MustOpenHolder()
|
||||
defer hldr.Close()
|
||||
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
|
||||
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1)
|
||||
|
||||
called := false
|
||||
e := NewExecutor(hldr.Holder, NewCluster(1))
|
||||
idx := e.Holder.Index("d")
|
||||
if idx == nil {
|
||||
t.Fatal("idex not found")
|
||||
}
|
||||
|
||||
idx.Stats = &MockStats{
|
||||
mockCount: func(name string, value int64) {
|
||||
if name != "SetProfileAttrs" {
|
||||
t.Errorf("Expected SetProfilepAttrs, Results %s", name)
|
||||
}
|
||||
|
||||
called = true
|
||||
return
|
||||
},
|
||||
}
|
||||
if _, err := e.Execute(context.Background(), "d", MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !called {
|
||||
t.Error("Count isn't called")
|
||||
}
|
||||
}
|
||||
|
||||
type MockStats struct {
|
||||
mockCount func(name string, value int64)
|
||||
mockCountWithTags func(name string, value int64, tags []string)
|
||||
}
|
||||
|
||||
func (s *MockStats) Count(name string, value int64) {
|
||||
if s.mockCount != nil {
|
||||
s.mockCount(name, value)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *MockStats) CountWithCustomTags(name string, value int64, tags []string) {
|
||||
if s.mockCountWithTags != nil {
|
||||
s.mockCountWithTags(name, value, tags)
|
||||
return
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (c *MockStats) Tags() []string { return nil }
|
||||
func (c *MockStats) WithTags(tags ...string) pilosa.StatsClient { return c }
|
||||
func (c *MockStats) Gauge(name string, value float64) {}
|
||||
func (c *MockStats) Histogram(name string, value float64) {}
|
||||
func (c *MockStats) Set(name string, value string) {}
|
||||
func (c *MockStats) Timing(name string, value time.Duration) {}
|
||||
32
view.go
32
view.go
|
|
@ -26,7 +26,7 @@ func IsValidView(name string) bool {
|
|||
type View struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
db string
|
||||
index string
|
||||
frame string
|
||||
name string
|
||||
|
||||
|
|
@ -38,15 +38,15 @@ type View struct {
|
|||
|
||||
stats StatsClient
|
||||
|
||||
BitmapAttrStore *AttrStore
|
||||
LogOutput io.Writer
|
||||
RowAttrStore *AttrStore
|
||||
LogOutput io.Writer
|
||||
}
|
||||
|
||||
// NewView returns a new instance of View.
|
||||
func NewView(path, db, frame, name string, cacheSize uint32) *View {
|
||||
func NewView(path, index, frame, name string, cacheSize uint32) *View {
|
||||
return &View{
|
||||
path: path,
|
||||
db: db,
|
||||
index: index,
|
||||
frame: frame,
|
||||
name: name,
|
||||
cacheSize: cacheSize,
|
||||
|
|
@ -62,8 +62,8 @@ func NewView(path, db, frame, name string, cacheSize uint32) *View {
|
|||
// Name returns the name the view was initialized with.
|
||||
func (v *View) Name() string { return v.name }
|
||||
|
||||
// DB returns the database name the view was initialized with.
|
||||
func (v *View) DB() string { return v.db }
|
||||
// Index returns the index name the view was initialized with.
|
||||
func (v *View) Index() string { return v.index }
|
||||
|
||||
// Frame returns the frame name the view was initialized with.
|
||||
func (v *View) Frame() string { return v.frame }
|
||||
|
|
@ -124,7 +124,7 @@ func (v *View) openFragments() error {
|
|||
if err := frag.Open(); err != nil {
|
||||
return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err)
|
||||
}
|
||||
frag.BitmapAttrStore = v.BitmapAttrStore
|
||||
frag.RowAttrStore = v.RowAttrStore
|
||||
v.fragments[frag.Slice()] = frag
|
||||
|
||||
v.stats.Count("maxSlice", 1)
|
||||
|
|
@ -205,7 +205,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
|
|||
if err := frag.Open(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
frag.BitmapAttrStore = v.BitmapAttrStore
|
||||
frag.RowAttrStore = v.RowAttrStore
|
||||
|
||||
// Save to lookup.
|
||||
v.fragments[slice] = frag
|
||||
|
|
@ -216,7 +216,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
|
|||
}
|
||||
|
||||
func (v *View) newFragment(path string, slice uint64) *Fragment {
|
||||
frag := NewFragment(path, v.db, v.frame, v.name, slice)
|
||||
frag := NewFragment(path, v.index, v.frame, v.name, slice)
|
||||
frag.cacheType = v.cacheType
|
||||
frag.cacheSize = v.cacheSize
|
||||
frag.LogOutput = v.LogOutput
|
||||
|
|
@ -225,23 +225,23 @@ func (v *View) newFragment(path string, slice uint64) *Fragment {
|
|||
}
|
||||
|
||||
// SetBit sets a bit within the view.
|
||||
func (v *View) SetBit(bitmapID, profileID uint64) (changed bool, err error) {
|
||||
slice := profileID / SliceWidth
|
||||
func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) {
|
||||
slice := columnID / SliceWidth
|
||||
frag, err := v.CreateFragmentIfNotExists(slice)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
return frag.SetBit(bitmapID, profileID)
|
||||
return frag.SetBit(rowID, columnID)
|
||||
}
|
||||
|
||||
// ClearBit clears a bit within the view.
|
||||
func (v *View) ClearBit(bitmapID, profileID uint64) (changed bool, err error) {
|
||||
slice := profileID / SliceWidth
|
||||
func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
|
||||
slice := columnID / SliceWidth
|
||||
frag, err := v.CreateFragmentIfNotExists(slice)
|
||||
if err != nil {
|
||||
return changed, err
|
||||
}
|
||||
return frag.ClearBit(bitmapID, profileID)
|
||||
return frag.ClearBit(rowID, columnID)
|
||||
}
|
||||
|
||||
// IsInverseView returns true if the view is used for storing an inverted representation.
|
||||
|
|
|
|||
36
view_test.go
36
view_test.go
|
|
@ -10,11 +10,11 @@ import (
|
|||
// View is a test wrapper for pilosa.View.
|
||||
type View struct {
|
||||
*pilosa.View
|
||||
BitmapAttrStore *AttrStore
|
||||
RowAttrStore *AttrStore
|
||||
}
|
||||
|
||||
// NewView returns a new instance of View with a temporary path.
|
||||
func NewView(db, frame, name string) *View {
|
||||
func NewView(index, frame, name string) *View {
|
||||
file, err := ioutil.TempFile("", "pilosa-view-")
|
||||
if err != nil {
|
||||
panic(err)
|
||||
|
|
@ -22,16 +22,16 @@ func NewView(db, frame, name string) *View {
|
|||
file.Close()
|
||||
|
||||
v := &View{
|
||||
View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize),
|
||||
BitmapAttrStore: MustOpenAttrStore(),
|
||||
View: pilosa.NewView(file.Name(), index, frame, name, pilosa.DefaultCacheSize),
|
||||
RowAttrStore: MustOpenAttrStore(),
|
||||
}
|
||||
v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore
|
||||
v.View.RowAttrStore = v.RowAttrStore.AttrStore
|
||||
return v
|
||||
}
|
||||
|
||||
// MustOpenView creates and opens an view at a temporary path. Panic on error.
|
||||
func MustOpenView(db, frame, name string) *View {
|
||||
v := NewView(db, frame, name)
|
||||
func MustOpenView(index, frame, name string) *View {
|
||||
v := NewView(index, frame, name)
|
||||
if err := v.Open(); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
|
@ -41,7 +41,7 @@ func MustOpenView(db, frame, name string) *View {
|
|||
// Close closes the view and removes all underlying data.
|
||||
func (v *View) Close() error {
|
||||
defer os.Remove(v.Path())
|
||||
defer v.BitmapAttrStore.Close()
|
||||
defer v.RowAttrStore.Close()
|
||||
return v.View.Close()
|
||||
}
|
||||
|
||||
|
|
@ -52,28 +52,28 @@ func (v *View) Reopen() error {
|
|||
return err
|
||||
}
|
||||
|
||||
v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize)
|
||||
v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore
|
||||
v.View = pilosa.NewView(path, v.Index(), v.Frame(), v.Name(), pilosa.DefaultCacheSize)
|
||||
v.View.RowAttrStore = v.RowAttrStore.AttrStore
|
||||
if err := v.Open(); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// MustSetBits sets bits on a bitmap. Panic on error.
|
||||
// MustSetBits sets bits on a row. Panic on error.
|
||||
// This function does not accept a timestamp or quantum.
|
||||
func (v *View) MustSetBits(bitmapID uint64, profileIDs ...uint64) {
|
||||
for _, profileID := range profileIDs {
|
||||
if _, err := v.SetBit(bitmapID, profileID); err != nil {
|
||||
func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) {
|
||||
for _, columnID := range columnIDs {
|
||||
if _, err := v.SetBit(rowID, columnID); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// MustClearBits clears bits on a bitmap. Panic on error.
|
||||
func (v *View) MustClearBits(bitmapID uint64, profileIDs ...uint64) {
|
||||
for _, profileID := range profileIDs {
|
||||
if _, err := v.ClearBit(bitmapID, profileID); err != nil {
|
||||
// MustClearBits clears bits on a row. Panic on error.
|
||||
func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) {
|
||||
for _, columnID := range columnIDs {
|
||||
if _, err := v.ClearBit(rowID, columnID); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue