scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 11:27:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor

Browse source 
This commit refactors the pilosa codebase. It makes several major
changes:

* Removes bitmap handles
* Removes dispatch/hold/transport
* Removes etcd dependency
* Adds consistent hash ring for slice placement
* Refactors parser/lexer
* Adds strong typing to PQL AST
* Flattens package hierarchy
This commit is contained in:
Ben Johnson 2015-09-23 17:23:38 -06:00
parent 467d51a3db
commit a632d08629
79 changed files with 4083 additions and 18028 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
Godeps/Godeps.json generated
View file

@ -1,6 +1,6 @@
{
"ImportPath": "github.com/umbel/pilosa",
"GoVersion": "go1.5",
"GoVersion": "go1.5.1",
"Packages": [
"./..."
],
@ -10,67 +10,21 @@
"Comment": "v0.1.0-21-g056c9bc",
"Rev": "056c9bc7be7190eaa7715723883caffa5f8fa3e4"
},
{
"ImportPath": "github.com/bitly/go-notify",
"Rev": "0a148b8111d688ba7550fc7119fe0d5d8e650838"
},
{
"ImportPath": "github.com/cactus/go-statsd-client/statsd",
"Rev": "3999011ef0451eb805b0fa31f98cfb9261bd71be"
},
{
"ImportPath": "github.com/cihub/seelog",
"Comment": "go1.1-81-gc510775",
"Rev": "c510775bb50d98213cfafca75a4bc5e3fddc8d8f"
},
{
"ImportPath": "github.com/coreos/go-etcd/etcd",
"Rev": "1e26d8ee84cf9b1000d2af8acfb45b2521f49be5"
},
{
"ImportPath": "github.com/davecgh/go-spew/spew",
"Rev": "e762b3d1320b76030bd7f6cc2bfc3d9acce874c0"
},
{
"ImportPath": "github.com/gocql/gocql",
"Comment": "1st_gen_framing-221-gf8fb76b",
"Rev": "f8fb76bb772442ea938e4be46ab1666e542411a5"
"ImportPath": "github.com/gogo/protobuf/proto",
"Rev": "499788908625f4d83de42a204d1350fde8588e4f"
},
{
"ImportPath": "github.com/golang/groupcache/lru",
"Rev": "d781998583680cda80cf61e0b37dd0cd8da2eb52"
},
{
"ImportPath": "github.com/golang/snappy",
"Rev": "723cc1e459b8eea2dea4583200fd60757d40097a"
},
{
"ImportPath": "github.com/gorilla/websocket",
"Rev": "92334662baa9cbebc2e6e68b8d56bc1233f85a4c"
},
{
"ImportPath": "github.com/kr/s3",
"Rev": "c070c8f9a8f0032d48f0d2a77d4e382788bd8a1d"
},
{
"ImportPath": "github.com/robertkrimen/otto",
"Rev": "f9e07770bd9b5142de517b05a85ffbb42d1895da"
},
{
"ImportPath": "github.com/stathat/consistent",
"Rev": "0262985f12333852b6423025aa3ca315af473a4a"
},
{
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
"Rev": "183614d6b32571e867df4cf086f5480ceefbdfac"
},
{
"ImportPath": "github.com/yasushi-saito/rbtree",
"Rev": "571e2538414bf914c7e2909b61217b4e3e5508f4"
},
{
"ImportPath": "gopkg.in/inf.v0",
"Rev": "c85f1217d51339c0fa3a498cc8b2075de695dae6"
}
]
}

26
NOTES Normal file
View file

@ -0,0 +1,26 @@
DB Profile
┌───────────▼────────────────────────────┐
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
Bitmap──▶0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
F ▶│0000000000000000000000000000000000000000│
r ││0000000000000000000000000000000000000000│
a ││0000000000000000000000000000000000000000│
m ││0000000000000000000000000000000000000000│
e ▶│0000000000000000000000000000000000000000│
└────────────────────────────────────────┘
▲───────────▲
Slice
Fragment=intersection of frame & slice

22
README.md
View file

@ -24,28 +24,12 @@ Now you can install the `pilosa` binary:
$ go install github.com/umbel/pilosa/...
```
Pilosa requires that [etcd][] is running locally for coordinating the cluster:
```sh
# In another terminal window
$ etcd
```
Now run `pilosa` with the default configuration:
```sh
pilosa
```
This setup assumes that cassandra and etcd are running locally and that
cassandra keyspace pilosa has been setup as describe in the file
`index/storage_cass.go`.
If you don't want backend storage you can comment out storage_backend in the
config file and it will just operate out of memory.
[etcd]: https://github.com/coreos/etcd
## Development
@ -68,9 +52,3 @@ $ godep save ./...
[godep]: https://github.com/tools/godep
### Running tests
Because of a bug in Go 1.5, you'll need to run `make test` to exclude tests
the the `vendor/` directory.

8796
benchmark_autogen_test.go
View file

File diff suppressed because it is too large Load diff

267
bitmap.go
View file

@ -8,8 +8,11 @@ import (
"encoding/base64"
"encoding/binary"
"encoding/gob"
"encoding/json"
"io"
log "github.com/cihub/seelog"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa/internal"
"github.com/yasushi-saito/rbtree"
)
@ -43,6 +46,15 @@ func (b *Bitmap) Chunk(c *Chunk) *Chunk {
// AddChunk adds c to the bitmap.
func (b *Bitmap) AddChunk(c *Chunk) { b.tree.Insert(c) }
// Chunks returns a list of all chunks.
func (b *Bitmap) Chunks() []*Chunk {
var a []*Chunk
for itr := b.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
a = append(a, itr.Item().Clone())
}
return a
}
// ChunkIterator returns an iterator for looping over the bitmap's chunks.
func (b *Bitmap) ChunkIterator() *ChunkIterator {
return &ChunkIterator{b.tree.Min()}
@ -58,18 +70,21 @@ func (b *Bitmap) Clone() *Bitmap {
break
}
node := itr.Item()
other.AddChunk(&Chunk{
Key: node.Key,
Value: node.Value.copy(),
})
other.AddChunk(itr.Item().Clone())
itr = itr.Next()
}
return other
}
// IntersectionCount returns the number of itersections between b and other.
// Merge adds chunks from other to b.
// Chunks in b are overwritten if they exist in other.
func (b *Bitmap) Merge(other *Bitmap) {
for itr := other.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
b.AddChunk(itr.Item().Clone())
}
}
// IntersectionCount returns the number of intersections between b and other.
func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
itr0 := b.ChunkIterator()
itr1 := other.ChunkIterator()
@ -91,8 +106,8 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
return results
}
// Intersection returns the itersection of b and other.
func (b *Bitmap) Intersection(other *Bitmap) *Bitmap {
// Intersect returns the itersection of b and other.
func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
itr0 := b.ChunkIterator()
itr1 := other.ChunkIterator()
@ -107,7 +122,7 @@ func (b *Bitmap) Intersection(other *Bitmap) *Bitmap {
} else if itr0.Item().Key == itr1.Item().Key {
output.AddChunk(&Chunk{
Key: itr0.Item().Key,
Value: itr0.Item().Value.intersection(itr1.Item().Value),
Value: itr0.Item().Value.intersect(itr1.Item().Value),
})
itr0 = itr0.Next()
itr1 = itr1.Next()
@ -202,7 +217,7 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
Value: itr0.Item().Value.difference(itr1.Item().Value),
}
// Could not add if all zero
// Do not add if all bits are zeroed.
if chunk.Value.bitcount() > 0 {
output.AddChunk(chunk)
}
@ -236,35 +251,87 @@ func (b *Bitmap) ToRawCompressString() (string, int) {
return base64.StdEncoding.EncodeToString(bt.Bytes()), max_slice
}
// ToBytes returns a gob-encoded byte slice of b.
func (b *Bitmap) ToBytes() []byte {
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
enc.Encode(b.tree.Len())
// WriteTo writes the encoded bitmap to w.
func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
// Wrap output in gzip compression.
z := gzip.NewWriter(w)
// Encode chunk count.
enc := gob.NewEncoder(w)
if err := enc.Encode(b.tree.Len()); err != nil {
return 0, err
}
// Encode all chunks.
for i := b.tree.Min(); !i.Limit(); i = i.Next() {
obj := i.Item().(*Chunk)
err := enc.Encode(obj)
if err != nil {
log.Warn(err)
if err := enc.Encode(i.Item().(*Chunk)); err != nil {
return 0, err
}
}
return buf.Bytes()
// Flush and close.
if err := z.Close(); err != nil {
return 0, err
}
return 0, nil
}
// FromBytes decodes a gob-encoded byte slice into b.
func (b *Bitmap) FromBytes(raw []byte) {
buf := bytes.NewBuffer(raw)
dec := gob.NewDecoder(buf)
// ReadFrom reads encoded bitmap data from r into b.
func (b *Bitmap) ReadFrom(r io.Reader) (n int64, err error) {
// Uncompress from gzip format.
z, err := gzip.NewReader(r)
if err != nil {
return 0, err
}
dec := gob.NewDecoder(z)
// Read size from data.
var size int
dec.Decode(&size)
if err := dec.Decode(&size); err != nil {
return 0, err
}
// Read chunks into bitmap.
b.tree = rbtree.NewTree(rbtreeItemCompare)
for i := 0; i < size; i++ {
var chunk Chunk
dec.Decode(&chunk)
if err := dec.Decode(&chunk); err != nil {
return 0, err
}
b.AddChunk(&chunk)
}
b.SetCount(b.BitCount())
return 0, nil
}
// MarshalJSON returns a JSON-encoded byte slice of b.
func (b *Bitmap) MarshalJSON() ([]byte, error) {
o := bitmapJSON{
Chunks: make([]chunkJSON, 0, b.tree.Len()),
}
for itr := b.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
o.Chunks = append(o.Chunks, chunkJSON{Key: itr.Item().Key, Value: itr.Item().Value})
}
return json.Marshal(&o)
}
// MarshalBinary returns a gob-encoded byte slice of b.
func (b *Bitmap) MarshalBinary() ([]byte, error) {
var buf bytes.Buffer
if _, err := b.WriteTo(&buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// UnmarshalBinary decodes a gob-encoded byte slice into b.
func (b *Bitmap) UnmarshalBinary(data []byte) error {
_, err := b.ReadFrom(bytes.NewReader(data))
return err
}
// Bits returns the bits in b as a slice of ints.
@ -291,7 +358,7 @@ func (b *Bitmap) Bits() []uint64 {
}
// SetBit sets the i-th bit of the bitmap.
func (b *Bitmap) SetBit(i uint64) (bool, *Chunk, Address) {
func (b *Bitmap) SetBit(i uint64) (changed bool) {
address := deref(i)
chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)})
@ -300,29 +367,29 @@ func (b *Bitmap) SetBit(i uint64) (bool, *Chunk, Address) {
b.AddChunk(chunk)
}
changed := chunk.Value.setBit(address.BlockIndex, address.Bit)
changed = chunk.Value.setBit(address.BlockIndex, address.Bit)
if changed {
b.bcount++
}
return changed, chunk, address
return changed
}
// ClearBit clears the i-th bit of the bitmap.
func (b *Bitmap) ClearBit(i uint64) (bool, *Chunk, Address) {
func (b *Bitmap) ClearBit(i uint64) (changed bool) {
address := deref(i)
chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)})
if chunk == nil {
return false, nil, address
return false
}
changed := chunk.Value.clearBit(address.BlockIndex, address.Bit)
changed = chunk.Value.clearBit(address.BlockIndex, address.Bit)
if changed && b.bcount > 0 {
b.bcount--
}
return changed, chunk, address
return changed
}
// Len returns the number of chunks in b.
@ -343,12 +410,75 @@ func (b *Bitmap) BitCount() uint64 {
return n
}
// encodeBitmap converts b into its internal representation.
func encodeBitmap(b *Bitmap) *internal.Bitmap {
pb := &internal.Bitmap{}
for i := b.tree.Min(); !i.Limit(); i = i.Next() {
pb.Chunks = append(pb.Chunks, encodeChunk(i.Item().(*Chunk)))
}
return pb
}
// decodeBitmap converts b from its internal representation.
func decodeBitmap(pb *internal.Bitmap) *Bitmap {
b := NewBitmap()
for _, chunk := range pb.GetChunks() {
b.AddChunk(decodeChunk(chunk))
}
b.SetCount(b.BitCount())
return b
}
// Union performs a union on a slice of bitmaps.
func Union(bitmaps []*Bitmap) *Bitmap {
other := bitmaps[0]
for _, bm := range bitmaps[1:] {
other = other.Union(bm)
}
return other
}
// bitmapJSON is the JSON representation of Bitmap.
type bitmapJSON struct {
Chunks []chunkJSON `json:"chunks"`
}
// Chunk represents a set of blocks in a Bitmap.
type Chunk struct {
Key uint64
Value Blocks
}
// Clone returns a copy of c.
func (c *Chunk) Clone() *Chunk {
return &Chunk{
Key: c.Key,
Value: c.Value.copy(),
}
}
// encodeChunks encodes c into its internal representation.
func encodeChunk(c *Chunk) *internal.Chunk {
return &internal.Chunk{
Key: proto.Uint64(c.Key),
Value: []uint64(c.Value),
}
}
// decodeChunk decodes c from its internal representation.
func decodeChunk(pb *internal.Chunk) *Chunk {
return &Chunk{
Key: pb.GetKey(),
Value: Blocks(pb.GetValue()),
}
}
// chunkJSON is the JSON representation of Chunk.
type chunkJSON struct {
Key uint64
Value []uint64
}
// ChunkIterator represents an object for iterating over chunks in a bitmap.
type ChunkIterator struct {
itr rbtree.Iterator
@ -383,6 +513,73 @@ func rbtreeItemCompare(a, b rbtree.Item) int {
return 0
}
type Blocks []uint64
// NewBlocks returns a 32-length Block.
func NewBlocks() Blocks {
return make(Blocks, 32)
}
func (a Blocks) bitcount() uint64 {
return popcntSlice(a)
}
func (a Blocks) union(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] | other[i]
}
return ret
}
func (a Blocks) invert() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = ^a[i]
}
return other
}
func (a Blocks) copy() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = a[i]
}
return other
}
func (a Blocks) andcount(other Blocks) uint64 {
return popcntAndSliceAsm(a, other)
}
func (a Blocks) intersect(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] & other[i]
}
return ret
}
func (a Blocks) difference(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] &^ other[i]
}
return ret
}
func (a Blocks) setBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] |= 1 << bit
return val == 0
}
func (a Blocks) clearBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] &= ^(1 << bit)
return val != 0
}
// Address represents a location for a given chunk/block/bit.
type Address struct {
ChunkKey uint64

74
blocks.go
View file

@ -1,74 +0,0 @@
package pilosa
import (
"fmt"
)
type Blocks []uint64
// NewBlocks returns a 32-length Block.
func NewBlocks() Blocks {
return make(Blocks, 32)
}
func (a Blocks) bitcount() uint64 {
fmt.Println("bitcount:", a)
return popcntSlice(a)
}
func (a Blocks) union(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] | other[i]
}
return ret
}
func (a Blocks) invert() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = ^a[i]
}
return other
}
func (a Blocks) copy() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = a[i]
}
return other
}
func (a Blocks) andcount(other Blocks) uint64 {
println("andcount")
return popcntAndSliceAsm(a, other)
}
func (a Blocks) intersection(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] & other[i]
}
return ret
}
func (a Blocks) difference(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] &^ other[i]
}
return ret
}
func (a Blocks) setBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] |= 1 << bit
return val == 0
}
func (a Blocks) clearBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] &= ^(1 << bit)
return val != 0
}

448
brand.go
View file

@ -1,448 +0,0 @@
package pilosa
import (
"encoding/json"
"fmt"
"math/rand"
"sort"
"sync"
"time"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa/statsd"
)
var FragmentBase string
var globalLock sync.Mutex
type Pair struct {
Key, Count uint64
}
type Rank struct {
*Pair
bitmap *Bitmap
category uint64
}
type RankList []*Rank
func (p RankList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p RankList) Len() int { return len(p) }
func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count }
type Brand struct {
bitmap_cache map[uint64]*Rank
db string
frame string
slice int
storage Storage
rankings RankList
rank_count int
threshold_value uint64
threshold_length int
threshold_idx int
skip int
rank_time time.Time
}
func NewBrand(db string, frame string, slice int, s Storage, threshold_len int, threshold int, skipp int) *Brand {
f := new(Brand)
f.storage = s
f.frame = frame
f.slice = slice
f.db = db
f.rank_count = 0
f.threshold_value = 0
f.threshold_length = threshold_len
f.threshold_idx = threshold
f.skip = skipp
f.Clear() //alloc the cache
return f
}
func (b *Brand) Clear() bool {
b.bitmap_cache = make(map[uint64]*Rank)
return true
}
func (b *Brand) Exists(bitmap_id uint64) bool {
_, ok := b.bitmap_cache[bitmap_id]
return ok
}
func (b *Brand) Get(bitmap_id uint64) *Bitmap {
if bm, ok := b.bitmap_cache[bitmap_id]; ok {
return bm.bitmap
}
// I should fetch the category here..need to come up with a good source
bm, filter := b.storage.Fetch(bitmap_id, b.db, b.frame, b.slice)
b.cache_it(bm, bitmap_id, filter)
return bm
}
func (b *Brand) Get_nocache(bitmap_id uint64) (*Bitmap, uint64) {
if bm, ok := b.bitmap_cache[bitmap_id]; ok {
return bm.bitmap, bm.category
}
//I should fetch the category here..need to come up with a good source
return b.storage.Fetch(bitmap_id, b.db, b.frame, b.slice)
}
func (b *Brand) GetFilter(bitmap_id, filter uint64) *Bitmap {
bm, old_filter := b.storage.Fetch(bitmap_id, b.db, b.frame, b.slice)
if filter == 0 {
filter = old_filter
}
b.cache_it(bm, bitmap_id, filter)
return bm
}
func (b *Brand) cache_it(bm *Bitmap, bitmap_id uint64, category uint64) {
if bm.Count() >= b.threshold_value {
b.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm, category}
if len(b.bitmap_cache) > b.threshold_length {
log.Info("RANK:", len(b.bitmap_cache), b.threshold_length, b.threshold_value)
b.Rank()
b.trim()
}
}
}
func (b *Brand) trim() {
for k, item := range b.bitmap_cache {
if item.bitmap.Count() <= b.threshold_value {
delete(b.bitmap_cache, k)
}
}
log.Info("TRIM:", len(b.bitmap_cache), b.threshold_length)
}
func (b *Brand) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool {
bm1, ok := b.bitmap_cache[bitmap_id]
var bm *Bitmap
if ok {
bm = bm1.bitmap
} else {
bm = b.GetFilter(bitmap_id, filter) //aways overwrites what is in cass filter type
}
change, chunk, address := bm.SetBit(bit_pos)
if change {
val := chunk.Value[address.BlockIndex]
b.storage.StoreBit(bitmap_id, b.db, b.frame, b.slice, filter, address.ChunkKey, int32(address.BlockIndex), val, bm.Count())
b.rank_count++
}
return change
}
func (b *Brand) Rank() {
start := time.Now()
var list RankList
for k, item := range b.bitmap_cache {
list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap, item.category})
}
sort.Sort(list)
b.rankings = list
if len(list) > b.threshold_idx {
item := list[b.threshold_idx]
b.threshold_value = item.bitmap.Count()
} else {
b.threshold_value = 1
}
b.rank_count = 0
delta := time.Since(start)
statsd.SendTimer("brand_Rank", delta.Nanoseconds())
b.rank_time = start
}
func packagePairs(r RankList) []Pair {
res := make([]Pair, r.Len())
for i, v := range r {
res[i] = Pair{v.Key, v.Count}
}
return res
}
func (b *Brand) Stats() interface{} {
total := uint64(0)
i := uint64(0)
bit_total := uint64(0)
for _, v := range b.bitmap_cache {
total += uint64(v.bitmap.Len()) * uint64(256)
i += 1
bit_total += v.Count
}
avg_bytes := uint64(0)
avg_bits := uint64(0)
if i > 0 {
avg_bytes = total / i
avg_bits = bit_total / i
}
stats := map[string]interface{}{
"total size of cache in bytes": total,
"number of bitmaps": len(b.bitmap_cache),
"avg size of bitmap in space(bytes)": avg_bytes,
"avg size of bitmap in bits": avg_bits,
"rank counter": b.rank_count,
"threshold_value": b.threshold_value,
"threshold_length": b.threshold_length,
"threshold_idx": b.threshold_idx,
"skip": b.skip}
return stats
}
func (b *Brand) Store(bitmap_id uint64, bm *Bitmap, filter uint64) error {
if err := b.storage.Store(bitmap_id, b.db, b.frame, b.slice, filter, bm); err != nil {
return err
}
b.cache_it(bm, bitmap_id, filter)
return nil
}
func (b *Brand) checkRank() {
if len(b.rankings) < 50 {
b.Rank()
return
}
if b.rank_count > 0 {
last := time.Since(b.rank_time) * time.Second
if last > 60*5 {
b.Rank()
}
}
}
func (b *Brand) TopN(src_bitmap *Bitmap, n int, categories []uint64) []Pair {
b.checkRank()
set := make(map[uint64]struct{})
for _, v := range categories {
set[v] = struct{}{}
}
return b.TopNCat(src_bitmap, n, set)
}
func dump(r RankList, n int) {
for i, v := range r {
log.Info(i, v)
if i > n {
return
}
}
}
func (b *Brand) TopNAll(n int, categories []uint64) []Pair {
log.Trace("TopNAll")
b.checkRank()
results := make([]Pair, 0, 0)
set := make(map[uint64]struct{})
needCat := false
for _, v := range categories {
set[v] = struct{}{}
needCat = true
}
count := 0
for _, pair := range b.rankings {
if needCat {
if _, ok := set[pair.category]; !ok {
continue
}
}
if count >= n {
break
}
if pair.Count > 0 {
results = append(results, Pair{pair.Key, pair.Count})
}
count++
}
return results
}
func (b *Brand) TopNCat(src_bitmap *Bitmap, n int, set map[uint64]struct{}) []Pair {
breakout := 1000
var (
o *Rank
results RankList
)
counter := 0
x := 0
needCat := (len(set) > 0)
for i, pair := range b.rankings {
if needCat {
if _, ok := set[pair.category]; !ok {
continue
}
}
if counter > n {
break
}
bc := src_bitmap.IntersectionCount(pair.bitmap)
if bc > 0 {
results = append(results, &Rank{&Pair{pair.Key, bc}, nil, pair.category})
counter = counter + 1
}
x = i
}
sort.Sort(results)
if counter < n {
return packagePairs(results)
}
end := len(results) - 1
o = results[end]
current_threshold := o.Count
if current_threshold <= 10 {
return packagePairs(results)
}
results = append(results, o)
for i := x + 1; i < len(b.rankings); i++ {
o = b.rankings[i]
if needCat {
if _, ok := set[o.category]; !ok {
continue
}
counter = counter + 1
} else {
counter = counter + 1
}
if counter > breakout {
break
}
//if o.Count < current_threshold { //done
//need something to do with the size of initianl bitmap
if o.Count < current_threshold { //done
break
}
bc := src_bitmap.IntersectionCount(o.bitmap)
if bc > current_threshold {
if results[end-1].Count > bc {
results[end] = &Rank{&Pair{o.Key, bc}, nil, o.category}
current_threshold = bc
} else {
results[end+1] = &Rank{&Pair{o.Key, bc}, nil, o.category}
sort.Sort(results)
o = results[end]
current_threshold = o.Count
}
}
}
return packagePairs(results[:end])
}
func (b *Brand) getFileName() string {
base := FragmentBase
if base == "" {
base = "."
}
return fmt.Sprintf("%s/%s.%s.%d.json", base, b.db, b.frame, b.slice)
}
func (b *Brand) Persist() error {
log.Info("Brand Persist:", b.getFileName())
b.storage.Flush()
asize := len(b.bitmap_cache)
if asize == 0 {
log.Warn("Nothing to save ", b.getFileName())
return nil
}
w, err := createFile(b.getFileName())
if err != nil {
log.Warn("Error opening outfile ", b.getFileName())
log.Warn(err)
return err
}
defer w.Close()
defer b.storage.Close()
var list RankList
for k, item := range b.bitmap_cache {
list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap, item.category})
}
sort.Sort(list)
results := make([]uint64, asize)
i := 0
for _, k := range list { // map[uint64]*Rank
results[i] = k.Key
i += 1
}
encoder := json.NewEncoder(w)
return encoder.Encode(results)
}
func (b *Brand) Load(f *Fragment) {
log.Warn("Brand Load")
time.Sleep(time.Duration(rand.Intn(32)) * time.Second) //trying to avoid mass cassandra hit
r, err := openFile(b.getFileName())
if err != nil {
log.Warn("NO Brand Init File:", b.getFileName())
return
}
dec := json.NewDecoder(r)
var keys []uint64
if err := dec.Decode(&keys); err != nil {
return
}
globalLock.Lock()
defer globalLock.Unlock()
// probaly need to get a etcd lock too someday
for _, k := range keys {
b.Get(k)
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) //trying to avoid mass cassandra hit
}
}
func (b *Brand) ClearBit(bitmap_id uint64, bit_pos uint64) bool {
log.Trace("ClearBit", bitmap_id, bit_pos)
bm1, ok := b.bitmap_cache[bitmap_id]
var bm *Bitmap
filter := uint64(0)
if ok {
bm = bm1.bitmap
filter = bm1.category
} else {
bm, filter = b.Get_nocache(bitmap_id)
if bm.Count() == 0 {
return false //nothing to unset
}
}
changed, chunk, address := bm.ClearBit(bit_pos)
if changed {
val := chunk.Value[address.BlockIndex]
if val == 0 {
b.storage.RemoveBit(bitmap_id, b.db, b.frame, b.slice, filter, address.ChunkKey, int32(address.BlockIndex), bm.Count())
} else {
b.storage.StoreBit(bitmap_id, b.db, b.frame, b.slice, filter, address.ChunkKey, int32(address.BlockIndex), val, bm.Count())
}
b.rank_count++
}
return changed
}

44
brand_test.go
View file

@ -1,44 +0,0 @@
package pilosa_test
import (
"math/rand"
"testing"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
"github.com/umbel/pilosa/storage/mem"
)
var (
size int
membrand *pilosa.Brand
cassbrand *pilosa.Brand
)
func init() {
size = 1000
statsd.Setup()
// SetupCassandra()
membrand = pilosa.NewBrand("db", "frame", 0, mem.NewStorage(), size, size, 0)
for i := uint64(0); i < uint64(size); i++ {
membrand.SetBit(i, 0, 1)
}
// cassbrand = NewBrand("db", "frame", 0, NewCassStorage(), size, size, 0)
// for i := uint64(0); i < uint64(size); i++ {
// cassbrand.SetBit(i, 0, 1)
// }
}
func benchmarkBrand(b *testing.B, size int, fill int, brand *pilosa.Brand) {
println(b.N)
for i := 0; i < b.N; i++ {
bid := rand.Int() % size
profile := uint64(i % fill)
brand.SetBit(uint64(bid), profile, 1)
}
}
func BenchmarkBrandMemSetBitL2(b *testing.B) { benchmarkBrand(b, size, 1024*64, membrand) }
func BenchmarkBrandCasSetBitL2(b *testing.B) { benchmarkBrand(b, size, 1024*64, cassbrand) }

262
cache.go Normal file
View file

@ -0,0 +1,262 @@
package pilosa
import (
"encoding/json"
"io"
"sort"
"time"
"github.com/gogo/protobuf/proto"
"github.com/golang/groupcache/lru"
"github.com/umbel/pilosa/internal"
)
// Cache represents a cache for bitmaps.
type Cache interface {
io.WriterTo
io.ReaderFrom
Add(bitmapID, category uint64, bm *Bitmap)
Get(bitmapID uint64) (bm *Bitmap, ok bool)
Len() int
// Updates the cache, if necessary.
Invalidate()
// Returns a list of all key/count pairs.
Pairs() []Pair
}
// LRUCache represents a least recently used Cache implemenation.
type LRUCache struct {
cache *lru.Cache
keys map[uint64]struct{}
}
// NewLRUCache returns a new instance of LRUCache.
func NewLRUCache(maxEntries int) *LRUCache {
c := &LRUCache{
cache: lru.New(maxEntries),
keys: make(map[uint64]struct{}),
}
c.cache.OnEvicted = c.onEvicted
return c
}
// Get returns a bitmap with a given id.
func (c *LRUCache) Add(bitmapID, category uint64, bm *Bitmap) {
c.cache.Add(bitmapID, bm)
c.keys[bitmapID] = struct{}{}
}
// Get returns a bitmap with a given id.
func (c *LRUCache) Get(bitmapID uint64) (bm *Bitmap, ok bool) {
value, ok := c.cache.Get(bitmapID)
if !ok {
return nil, false
}
return value.(*Bitmap), true
}
// Len returns the number of items in the cache.
func (c *LRUCache) Len() int { return c.cache.Len() }
// Invalidate is a no-op.
func (c *LRUCache) Invalidate() {}
// Pairs returns all key/count pairs in the cache.
func (c *LRUCache) Pairs() []Pair {
a := make([]Pair, 0, len(c.keys))
for k := range c.keys {
bm, _ := c.Get(k)
a = append(a, Pair{
Key: k,
Count: bm.Count(),
})
}
return a
}
// WriteTo writes the cache to w.
func (c *LRUCache) WriteTo(w io.Writer) (n int64, err error) {
// Write keys to slice.
a := make([]uint64, 0, len(c.keys))
for k := range c.keys {
a = append(a, k)
}
// Encode to file as array of keys.
if err := json.NewEncoder(w).Encode(a); err != nil {
return 0, err
}
return 0, nil
}
// ReadFrom read from r into the cache.
func (c *LRUCache) ReadFrom(r io.Reader) (n int64, err error) {
var keys []uint64
if err := json.NewDecoder(r).Decode(&keys); err != nil {
return 0, err
}
panic("FIXME: TODO")
}
func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) {
delete(c.keys, key.(uint64))
}
// Ensure LRUCache implements Cache.
var _ Cache = &LRUCache{}
// RankCache represents a cache with sorted entries.
type RankCache struct {
entries map[uint64]*Pair
rankings []Pair // cached, ordered list
updateN int
updateTime time.Time
ThresholdLength int
ThresholdIndex int
ThresholdValue uint64
}
// NewRankCache returns a new instance of RankCache.
func NewRankCache() *RankCache {
return &RankCache{
entries: make(map[uint64]*Pair),
}
}
// Get returns a bitmap with a given id.
func (c *RankCache) Add(bitmapID, category uint64, bm *Bitmap) {
// Ignore if the bit count on the bitmap is below the threshold.
if bm.Count() < c.ThresholdValue {
return
}
// Add to cache.
c.entries[bitmapID] = &Pair{
Key: bitmapID,
Count: bm.Count(),
bitmap: bm,
category: category,
}
// If size is larger than the threshold then trim it.
if len(c.entries) > c.ThresholdLength {
c.update()
for k, entry := range c.entries {
if entry.bitmap.Count() <= c.ThresholdValue {
delete(c.entries, k)
}
}
}
}
// Get returns a bitmap with a given id.
func (c *RankCache) Get(bitmapID uint64) (bm *Bitmap, ok bool) {
entry, ok := c.entries[bitmapID]
if !ok {
return nil, false
}
return entry.bitmap, true
}
// Len returns the number of items in the cache.
func (c *RankCache) Len() int { return len(c.entries) }
// Invalidate reorders the entries, if necessary.
func (c *RankCache) Invalidate() {
// Update if there aren't many items or it hasn't been updated recently.
if len(c.rankings) < 50 || (c.updateN > 0 && time.Since(c.updateTime) > 5*time.Minute) {
c.update()
}
}
// update reorders the entries by rank.
func (c *RankCache) update() {
// Convert cache to a sorted list.
list := make([]Pair, 0, len(c.entries))
for k, item := range c.entries {
list = append(list, Pair{
Key: k,
Count: item.bitmap.Count(),
bitmap: item.bitmap,
category: item.category,
})
}
sort.Sort(Pairs(list))
// Store the count of the item at the threshold index.
c.rankings = list
if len(c.rankings) > c.ThresholdIndex {
c.ThresholdValue = list[c.ThresholdIndex].bitmap.Count()
} else {
c.ThresholdValue = 1
}
// Reset counters.
c.updateTime, c.updateN = time.Now(), 0
}
// Pairs returns an ordered list of key/count pairs.
func (c *RankCache) Pairs() []Pair { return c.rankings }
// WriteTo writes the cache to w.
func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) {
panic("FIXME: TODO")
}
// ReadFrom read from r into the cache.
func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
panic("FIXME: TODO")
}
// Ensure RankCache implements Cache.
var _ Cache = &RankCache{}
type Pair struct {
Key uint64 `json:"key"`
Count uint64 `json:"count"`
bitmap *Bitmap
category uint64
}
func encodePair(p Pair) *internal.Pair {
return &internal.Pair{
Key: proto.Uint64(p.Key),
Count: proto.Uint64(p.Count),
}
}
func decodePair(pb *internal.Pair) Pair {
return Pair{
Key: pb.GetKey(),
Count: pb.GetCount(),
}
}
type Pairs []Pair
func (p Pairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p Pairs) Len() int { return len(p) }
func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
func encodePairs(a Pairs) []*internal.Pair {
other := make([]*internal.Pair, len(a))
for i := range a {
other[i] = encodePair(a[i])
}
return other
}
func decodePairs(a []*internal.Pair) []Pair {
other := make([]Pair, len(a))
for i := range a {
other[i] = decodePair(a[i])
}
return other
}

123
cluster.go Normal file
View file

@ -0,0 +1,123 @@
package pilosa
import (
"encoding/binary"
"hash/fnv"
)
const (
// DefaultPartitionN is the default number of partitions in a cluster.
DefaultPartitionN = 64
// DefaultReplicaN is the default number of replicas per partition.
DefaultReplicaN = 1
)
// Node represents a node in the cluster.
type Node struct {
Host string
}
// Nodes represents a list of nodes.
type Nodes []*Node
// ContainsHost returns true if host matches on of the node's host.
func (a Nodes) ContainsHost(host string) bool {
for _, n := range a {
if n.Host == host {
return true
}
}
return false
}
// Hosts returns a list of all hostnames.
func (a Nodes) Hosts() []string {
hosts := make([]string, len(a))
for i, n := range a {
hosts[i] = n.Host
}
return hosts
}
// Cluster represents a collection of nodes.
type Cluster struct {
Nodes []*Node
// Hashing algorithm used to assign partitions to nodes.
Hasher Hasher
// The number of partitions in the cluster.
PartitionN int
// The number of replicas a partition has.
ReplicaN int
}
// NewCluster returns a new instance of Cluster with defaults.
func NewCluster() *Cluster {
return &Cluster{
Hasher: &jmphasher{},
PartitionN: DefaultPartitionN,
ReplicaN: DefaultReplicaN,
}
}
// Partition returns the partition that a slice belongs to.
func (c *Cluster) Partition(slice uint64) int {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], slice)
// Hash the bytes and mod by partition count.
h := fnv.New64a()
h.Write(buf[:])
return int(h.Sum64() % uint64(c.PartitionN))
}
// SliceNodes returns a list of nodes that own a slice.
func (c *Cluster) SliceNodes(slice uint64) []*Node {
return c.PartitionNodes(c.Partition(slice))
}
// PartitionNodes returns a list of nodes that own a partition.
func (c *Cluster) PartitionNodes(partitionID int) []*Node {
// Default replica count to between one and the number of nodes.
// The replica count can be zero if there are no nodes.
replicaN := c.ReplicaN
if replicaN > len(c.Nodes) {
replicaN = len(c.Nodes)
} else if replicaN == 0 {
replicaN = 1
}
// Determine primary owner node.
index := c.Hasher.Hash(uint64(partitionID), len(c.Nodes))
// Collect nodes around the ring.
nodes := make([]*Node, replicaN)
for i := 0; i < replicaN; i++ {
nodes[i] = c.Nodes[(index+i)%len(c.Nodes)]
}
return nodes
}
// Hasher represents an interface to hash integers into buckets.
type Hasher interface {
// Hashes the key into a number between [0,N).
Hash(key uint64, n int) int
}
// jmphasher represents an implementation of jmphash. Implements Hasher.
type jmphasher struct{}
// Hash returns the integer hash for the given key.
func (h *jmphasher) Hash(key uint64, n int) int {
b, j := int64(-1), int64(0)
for j < int64(n) {
b = j
key = key*uint64(2862933555777941757) + 1
j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1)))
}
return int(b)
}

90
cluster_test.go Normal file
View file

@ -0,0 +1,90 @@
package pilosa_test
import (
"fmt"
"math/rand"
"reflect"
"testing"
"testing/quick"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
)
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := pilosa.Cluster{
Nodes: []*pilosa.Node{
{Host: "serverA:1000"},
{Host: "serverB:1000"},
{Host: "serverC:1000"},
},
Hasher: NewModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(slice uint64, partitionN int) bool {
c := pilosa.NewCluster()
c.PartitionN = partitionN
partitionID := c.Partition(slice)
if partitionID < 0 || partitionID > partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
}
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0] = reflect.ValueOf(uint64(rand.Uint32()))
values[1] = reflect.ValueOf(rand.Intn(1000) + 1)
},
}); err != nil {
t.Fatal(err)
}
}
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewCluster(n int) *pilosa.Cluster {
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
Host: fmt.Sprintf("host%d", i),
})
}
return c
}
// ModHasher represents a simple, mod-based hashing.
type ModHasher struct{}
// NewModHasher returns a new instance of ModHasher with n buckets.
func NewModHasher() *ModHasher { return &ModHasher{} }
func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
// ConstHasher represents hash that always returns the same index.
type ConstHasher struct {
i int
}
// NewConstHasher returns a new instance of ConstHasher that always returns i.
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }

111
cmd/pilosa/config.go
View file

@ -4,106 +4,55 @@ import (
"time"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
_ "github.com/umbel/pilosa/storage"
"github.com/umbel/pilosa/storage/cassandra"
"github.com/umbel/pilosa/transport"
)
const (
// DefaultLogPath is the default file path where the log will be written.
DefaultLogPath = "/tmps"
// DefaultHost is the default hostname to use.
DefaultHost = "localhost"
// DefaultLogLevel is the default logging level used by seelog.
DefaultLogLevel = "info"
// DefaultFragmentBase is the default path where fragments are stored.
DefaultFragmentBase = "/tmp/single"
)
var (
// DefaultSupportedFrames are the frames supported by default.
DefaultSupportedFrames = [...]string{"b.n", "t.t", "l.n", "d", "p.n"}
// DefaultETCDHosts are the default hosts to connect to.
DefaultETCDHosts = [...]string{"http://127.0.0.1:4001"}
// DefaultAddr is the default HTTP address to use.
DefaultAddr = ":15000"
)
// Config represents the configuration for the command.
type Config struct {
ID *pilosa.GUID `toml:"id"`
Host string `toml:"host"`
Host string `toml:"host"`
Addr string `toml:"addr"`
TCP struct {
Port int `toml:"port"`
} `toml:"tcp"`
HTTP struct {
Port int `toml:"port"`
DefaultDB string `toml:"default-db"`
RequestLogPath string `toml:"request-log-path"`
SetBitLogEnabled bool `toml:"set-bit-log-enabled"`
} `toml:"http"`
Log struct {
Path string `toml:"path"`
Level string `toml:"level"`
}
Storage struct {
Backend string `toml:"backend"`
Hosts []string `toml:"hosts"`
Keyspace string `toml:"keyspace"`
FragmentBase string `toml:"fragment-base"`
SupportedFrames []string `toml:"supported-frames"`
CassandraTimeWindow Duration `toml:"cassandra-time-window"`
CassandraMaxSizeBatch int `toml:"cassandra-max-size-batch"`
} `toml:"storage"`
AWS struct {
AccessKeyID string `toml:"access-key-id"`
SecretAccessKey string `toml:"secret-access-key"`
} `toml:"aws"`
LevelDB struct {
Path string `toml:"path"`
} `toml:"leveldb"`
Statsd struct {
Host string `toml:"host"`
} `toml:"statsd"`
Cluster struct {
ReplicaN int `toml:"replicas"`
Nodes []struct {
Host string `toml:"host"`
} `toml:"nodes"`
} `toml:"cluster"`
Plugins struct {
Path string `toml:"path"`
} `toml:"plugins"`
ETCD struct {
Hosts []string `toml:"hosts"`
FragmentAllocLockTTL Duration `toml:"fragment-alloc-lock-ttl"`
} `toml:"etcd"`
}
// NewConfig returns an instance of Config with default options.
func NewConfig() Config {
var c Config
c.Host = "localhost"
c.TCP.Port = transport.DefaultTCPPort
c.HTTP.Port = transport.DefaultHTTPPort
c.Log.Path = DefaultLogPath
c.Storage.Backend = pilosa.DefaultBackend
c.Storage.Hosts = cassandra.DefaultHosts
c.Storage.Keyspace = cassandra.DefaultKeyspace
c.Storage.FragmentBase = DefaultFragmentBase
c.Storage.SupportedFrames = DefaultSupportedFrames[:]
c.Storage.CassandraTimeWindow = Duration(cassandra.DefaultFlushInterval)
c.Storage.CassandraMaxSizeBatch = cassandra.DefaultFlushThreshold
c.Statsd.Host = statsd.DefaultHost
c.ETCD.Hosts = DefaultETCDHosts[:]
func NewConfig() *Config {
c := &Config{
Host: DefaultHost,
Addr: DefaultAddr,
}
c.Cluster.ReplicaN = pilosa.DefaultReplicaN
return c
}
// PilosaCluster returns a new instance of pilosa.Cluster based on the config.
func (c *Config) PilosaCluster() *pilosa.Cluster {
cluster := pilosa.NewCluster()
cluster.ReplicaN = c.Cluster.ReplicaN
for _, n := range c.Cluster.Nodes {
cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: n.Host})
}
return cluster
}
// Duration is a TOML wrapper type for time.Duration.
type Duration time.Duration

159
cmd/pilosa/config_test.go
View file

@ -1,157 +1,27 @@
package main_test
import (
"reflect"
"testing"
"time"
"github.com/BurntSushi/toml"
"github.com/umbel/pilosa/cmd/pilosa"
)
// Ensure the ID can be parsed as a GUID.
func TestConfig_Parse_ID(t *testing.T) {
if c, err := ParseConfig(`
id = "00000000-0000-0000-0000-000000000001"
`); err != nil {
t.Fatal(err)
} else if c.ID == nil || c.ID.String() != `00000000-0000-0000-0000-000000000001` {
t.Fatalf("unexpected id: %s", c.ID)
}
}
// Ensure that parsing an invalid GUID returns an error.
func TestConfig_Parse_ID_ErrInvalid(t *testing.T) {
if _, err := ParseConfig(`
id = "x"
`); err == nil || err.Error() != `Type mismatch for 'main.Config.id': invalid GUID "x"` {
t.Fatal(err)
}
}
// Ensure the host can be parsed.
func TestConfig_Parse_Host(t *testing.T) {
if c, err := ParseConfig(`host = "localhost"`); err != nil {
if c, err := ParseConfig(`host = "local"`); err != nil {
t.Fatal(err)
} else if c.Host != "localhost" {
} else if c.Host != "local" {
t.Fatalf("unexpected host: %s", c.Host)
}
}
// Ensure the "tcp" config can be parsed.
func TestConfig_Parse_TCP(t *testing.T) {
if c, err := ParseConfig(`
[tcp]
port = 123
`); err != nil {
// Ensure the addr can be parsed.
func TestConfig_Parse_Addr(t *testing.T) {
if c, err := ParseConfig(`addr = ":80"`); err != nil {
t.Fatal(err)
} else if c.TCP.Port != 123 {
t.Fatalf("unexpected port: %s", c.TCP.Port)
}
}
// Ensure the "http" config can be parsed.
func TestConfig_Parse_HTTP(t *testing.T) {
if c, err := ParseConfig(`
[http]
port = 123
default-db = "xyz"
request-log-path = "/path/to/log"
set-bit-log-enabled = true
`); err != nil {
t.Fatal(err)
} else if c.HTTP.Port != 123 {
t.Fatalf("unexpected port: %s", c.HTTP.Port)
} else if c.HTTP.DefaultDB != "xyz" {
t.Fatalf("unexpected default db: %s", c.HTTP.DefaultDB)
} else if c.HTTP.RequestLogPath != "/path/to/log" {
t.Fatalf("unexpected request log path: %s", c.HTTP.RequestLogPath)
} else if c.HTTP.SetBitLogEnabled != true {
t.Fatalf("unexpected set bit log enabled: %v", c.HTTP.SetBitLogEnabled)
}
}
// Ensure the "log" config can be parsed.
func TestConfig_Parse_Log(t *testing.T) {
if c, err := ParseConfig(`
[log]
path = "/path/to/log"
level = "debug"
`); err != nil {
t.Fatal(err)
} else if c.Log.Path != "/path/to/log" {
t.Fatalf("unexpected path: %s", c.Log.Path)
} else if c.Log.Level != "debug" {
t.Fatalf("unexpected level: %s", c.Log.Level)
}
}
// Ensure the "storage" config can be parsed.
func TestConfig_Parse_Storage(t *testing.T) {
if c, err := ParseConfig(`
[storage]
backend = "cassandra"
hosts = ["server0", "server1"]
keyspace = "pilosa"
fragment-base = "/path/to/base"
supported-frames = ["a", "b", "c"]
cassandra-time-window = "5s"
cassandra-max-size-batch = 50
`); err != nil {
t.Fatal(err)
} else if c.Storage.Backend != "cassandra" {
t.Fatalf("unexpected backend: %s", c.Storage.Backend)
} else if !reflect.DeepEqual(c.Storage.Hosts, []string{"server0", "server1"}) {
t.Fatalf("unexpected hosts: %+v", c.Storage.Hosts)
} else if c.Storage.Keyspace != "pilosa" {
t.Fatalf("unexpected keyspace: %s", c.Storage.Keyspace)
} else if c.Storage.FragmentBase != "/path/to/base" {
t.Fatalf("unexpected fragment base: %s", c.Storage.FragmentBase)
} else if !reflect.DeepEqual(c.Storage.SupportedFrames, []string{"a", "b", "c"}) {
t.Fatalf("unexpected supported frames: %s", c.Storage.SupportedFrames)
} else if time.Duration(c.Storage.CassandraTimeWindow) != 5*time.Second {
t.Fatalf("unexpected cassandra time window: %s", time.Duration(c.Storage.CassandraTimeWindow))
} else if c.Storage.CassandraMaxSizeBatch != 50 {
t.Fatalf("unexpected cassandra max size batch: %s", c.Storage.CassandraMaxSizeBatch)
}
}
// Ensure the "aws" config can be parsed.
func TestConfig_Parse_AWS(t *testing.T) {
if c, err := ParseConfig(`
[aws]
access-key-id = "abc"
secret-access-key = "def"
`); err != nil {
t.Fatal(err)
} else if c.AWS.AccessKeyID != "abc" {
t.Fatalf("unexpected access key id: %s", c.AWS.AccessKeyID)
} else if c.AWS.SecretAccessKey != "def" {
t.Fatalf("unexpected secret access key: %s", c.AWS.SecretAccessKey)
}
}
// Ensure the "leveldb" config can be parsed.
func TestConfig_Parse_LevelDB(t *testing.T) {
if c, err := ParseConfig(`
[leveldb]
path = "/path/to/db"
`); err != nil {
t.Fatal(err)
} else if c.LevelDB.Path != "/path/to/db" {
t.Fatalf("unexpected path: %s", c.LevelDB.Path)
}
}
// Ensure the "statsd" config can be parsed.
func TestConfig_Parse_Statsd(t *testing.T) {
if c, err := ParseConfig(`
[statsd]
host = "localhost"
`); err != nil {
t.Fatal(err)
} else if c.Statsd.Host != "localhost" {
t.Fatalf("unexpected host: %s", c.Statsd.Host)
} else if c.Addr != ":80" {
t.Fatalf("unexpected addr: %s", c.Addr)
}
}
@ -167,21 +37,6 @@ path = "/path/to/plugins"
}
}
// Ensure the "etcd" config can be parsed.
func TestConfig_Parse_ETCD(t *testing.T) {
if c, err := ParseConfig(`
[etcd]
hosts = ["127.0.0.1"]
fragment-alloc-lock-ttl = "5m"
`); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(c.ETCD.Hosts, []string{"127.0.0.1"}) {
t.Fatalf("unexpected hosts: %+v", c.ETCD.Hosts)
} else if time.Duration(c.ETCD.FragmentAllocLockTTL) != 5*time.Minute {
t.Fatalf("unexpected fragment alloc lock ttl: %v", c.ETCD.FragmentAllocLockTTL)
}
}
// ParseConfig parses s into a config.
func ParseConfig(s string) (main.Config, error) {
var c main.Config

229
cmd/pilosa/main.go
View file

@ -4,37 +4,64 @@ import (
"flag"
"fmt"
"io"
"log"
"math/rand"
"net"
"net/http"
"os"
"runtime/pprof"
"time"
"github.com/BurntSushi/toml"
log "github.com/cihub/seelog"
"github.com/coreos/go-etcd/etcd"
"github.com/kr/s3/s3util"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/dispatch"
"github.com/umbel/pilosa/executor"
"github.com/umbel/pilosa/hold"
"github.com/umbel/pilosa/statsd"
"github.com/umbel/pilosa/transport"
)
// Build holds the build information passed in at compile time.
var Build string
func init() {
if Build == "" {
Build = "v0.0.0"
}
rand.Seed(time.Now().UTC().UnixNano())
}
func main() {
m := NewMain()
if err := m.Run(os.Args[1:]...); err != nil {
fmt.Fprintln(m.Stderr, err.Error())
os.Exit(-1)
fmt.Fprintf(m.Stderr, "Pilosa %s\n", Build)
// Parse command line arguments.
if err := m.ParseFlags(os.Args[1:]); err != nil {
fmt.Fprintln(m.Stderr, err)
os.Exit(2)
}
// Execute the program.
if err := m.Run(); err != nil {
fmt.Fprintln(m.Stderr, err)
os.Exit(1)
}
// Wait indefinitely.
<-(chan struct{})(nil)
}
// Main represents the main program execution.
type Main struct {
ln net.Listener
// Path to the configuration file.
ConfigPath string
// Configuration options.
Config *Config
// Profiling paths
CPUProfile string
// Standard input/output
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
@ -42,6 +69,9 @@ type Main struct {
// NewMain returns a new instance of Main.
func NewMain() *Main {
return &Main{
Config: NewConfig(),
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
@ -49,31 +79,16 @@ func NewMain() *Main {
// Run executes the main program execution.
func (m *Main) Run(args ...string) error {
defer log.Flush()
logger := log.New(m.Stderr, "", log.LstdFlags)
// Parse command line arguments.
opt, err := m.ParseFlags(args)
if err != nil {
return err
}
// Parse configuration.
config := NewConfig()
if opt.ConfigPath != "" {
if _, err := toml.DecodeFile(opt.ConfigPath, &config); err != nil {
return err
}
}
// Generate an ID if one is not specified in the config.
id := config.ID
if id == nil {
*id = pilosa.NewGUID()
// Notify user of config file.
if m.ConfigPath != "" {
fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath)
}
// Set up profiling.
if opt.CPUProfile != "" {
f, err := os.Create(opt.CPUProfile)
if m.CPUProfile != "" {
f, err := os.Create(m.CPUProfile)
if err != nil {
return err
}
@ -82,127 +97,61 @@ func (m *Main) Run(args ...string) error {
defer pprof.StopCPUProfile()
}
// Pass configuration to packages.
// NOTE: This is temporary. These config options should be encapsulated in the types.
db.SupportedFrames = config.Storage.SupportedFrames
pilosa.FragmentBase = config.Storage.FragmentBase
pilosa.Backend = config.Storage.Backend
pilosa.LevelDBPath = config.LevelDB.Path
// Build cluster from config file.
cluster := m.Config.PilosaCluster()
// Initialize AWS storage.
s3util.DefaultConfig.AccessKey = config.AWS.AccessKeyID
s3util.DefaultConfig.SecretKey = config.AWS.SecretAccessKey
// Create index to store fragments.
index := pilosa.NewIndex()
// Initialize Statsd.
statsd.Host = config.Statsd.Host
statsd.Setup()
// Create executor for executing queries.
e := pilosa.NewExecutor(index)
e.Host = m.Config.Host
e.Cluster = cluster
// Initialize logging.
logger, _ := log.LoggerFromConfigAsBytes([]byte(SeelogProductionConfig(config.Log.Path, *id, config.Log.Level)))
log.ReplaceLogger(logger)
// Initialize HTTP handler.
h := pilosa.NewHandler()
h.Executor = e
h.LogOutput = m.Stderr
// Initialize etcd client.
etcdClient := etcd.NewClient(config.ETCD.Hosts)
// Open HTTP listener.
ln, err := net.Listen("tcp", m.Config.Addr)
if err != nil {
return err
}
m.ln = ln
// Initialize the cluster.
cluster := db.NewCluster()
// Serve HTTP.
go func() { logger.Print(http.Serve(ln, h)) }()
// Create index.
idx := pilosa.NewFragmentContainer()
// Initialize the holder.
hold := hold.NewHolder()
// Initialize process map.
processMap := core.NewProcessMap()
// Start process mapper.
processMapper := core.NewProcessMapper("/pilosa/0")
processMapper.ID = *id
processMapper.TCPPort = config.TCP.Port
processMapper.HTTPPort = config.HTTP.Port
processMapper.Host = config.Host
processMapper.ProcessMap = processMap
processMapper.EtcdClient = etcdClient
// Start topology mapper.
topologyMapper := core.NewTopologyMapper("/pilosa/0")
topologyMapper.Cluster = cluster
topologyMapper.ProcessMap = processMap
topologyMapper.EtcdClient = etcdClient
topologyMapper.Index = idx
topologyMapper.SupportedFrames = config.Storage.SupportedFrames
topologyMapper.FragmentAllocLockTTL = time.Duration(config.ETCD.FragmentAllocLockTTL)
// Start the transport.
transport := transport.NewTcpTransport(*id)
transport.Port = config.TCP.Port
transport.ProcessMap = processMap
go transport.Run()
// Create the pinger.
pinger := core.NewPinger(*id)
pinger.Hold = hold
pinger.Transport = transport
// Create the batcher.
batcher := core.NewBatcher(*id)
batcher.Cluster = cluster
batcher.Hold = hold
batcher.Transport = transport
// Start the web service.
core.RequestLogPath = config.HTTP.RequestLogPath
ws := core.NewWebService()
ws.ID = *id
ws.Port = config.HTTP.Port
ws.Version = Build
ws.DefaultDB = config.HTTP.DefaultDB
ws.SetBitLogEnabled = config.HTTP.SetBitLogEnabled
ws.Cluster = cluster
ws.TopologyMapper = topologyMapper
ws.Pinger = pinger
ws.Batcher = batcher
// Start the executor.
ex := executor.NewExecutor(*id)
ex.ProcessMap = processMap
ex.PluginsPath = config.Plugins.Path
ex.Hold = hold
ex.Index = idx
// Start the dispatcher.
dispatch := dispatch.NewDispatch()
dispatch.Executor = ex
dispatch.Hold = hold
dispatch.Index = idx
dispatch.Transport = transport
go dispatch.Run()
fmt.Printf("Pilosa %s\n", Build)
log.Warn("STOP")
fmt.Fprintf(m.Stderr, "Listening on http://%s\n", ln.Addr().String())
return nil
}
// Close shuts down the process.
func (m *Main) Close() error {
if m.ln != nil {
m.ln.Close()
}
return nil
}
// ParseFlags parses command line flags from args.
func (m *Main) ParseFlags(args []string) (Options, error) {
var opt Options
func (m *Main) ParseFlags(args []string) error {
fs := flag.NewFlagSet("pilosa", flag.ContinueOnError)
fs.SetOutput(m.Stderr)
fs.StringVar(&opt.ConfigPath, "config", "", "config path")
fs.StringVar(&opt.CPUProfile, "cpuprofile", "", "write cpu profile to file")
fs.StringVar(&m.ConfigPath, "config", "", "config path")
fs.StringVar(&m.CPUProfile, "cpuprofile", "", "write cpu profile to file")
if err := fs.Parse(args); err != nil {
return opt, err
return err
}
return opt, nil
}
// Load config, if specified.
if m.ConfigPath != "" {
if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil {
return err
}
}
// Options represents the command line options.
type Options struct {
CPUProfile string
ConfigPath string
return nil
}

28
cmd/pilosa/seelog.go
View file

@ -1,28 +0,0 @@
package main
import (
"fmt"
"github.com/umbel/pilosa"
)
func SeelogProductionConfig(path string, id pilosa.GUID, level string) string {
if path == "" {
path = "/tmp"
}
fname := fmt.Sprintf("%s/pilosa.%s", path, id.String())
//<seelog minlevel="debug" maxlevel="error">
s := fmt.Sprintf(`<seelog minlevel="%s">
<outputs>
<rollingfile type="size" filename="%s" maxsize="524288000" maxrolls="4" formatid="format1" />
</outputs> `, level, fname)
s += `<formats>
<format id="format1" format="%Date/%Time [%LEV] %Msg%n"/>
</formats>
</seelog>`
return s
}

65
cmd/pilosactl/main.go Normal file
View file

@ -0,0 +1,65 @@
package main
import (
"errors"
"flag"
"fmt"
"io"
"os"
)
func main() {
m := NewMain()
// Parse command line arguments.
if err := m.ParseFlags(os.Args[1:]); err != nil {
fmt.Fprintln(m.Stderr, err)
os.Exit(2)
}
// Execute the program.
if err := m.Run(); err != nil {
fmt.Fprintln(m.Stderr, err)
os.Exit(1)
}
}
// Main represents the main program execution.
type Main struct {
Command string
// Standard input/output
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
// NewMain returns a new instance of Main.
func NewMain() *Main {
return &Main{
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
}
// Run executes the main program execution.
func (m *Main) Run() error {
panic("FIXME: implement commands")
}
// ParseFlags parses command line flags from args.
func (m *Main) ParseFlags(args []string) error {
fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
fs.SetOutput(m.Stderr)
if err := fs.Parse(args); err != nil {
return err
}
// Require command.
if fs.NArg() == 0 {
return errors.New("command required")
}
return nil
}

1
cmd/pilosactl/main_test.go Normal file
View file

@ -0,0 +1 @@
package main_test

70
core/batch.go
View file

@ -1,70 +0,0 @@
package core
import (
"encoding/gob"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/hold"
)
type BatchRequest struct {
Id *pilosa.GUID
Source *pilosa.GUID
Fragment_id pilosa.SUUID
Bitmap_id uint64
Compressed_bitmap string
Filter uint64
}
type BatchResponse struct {
Id *pilosa.GUID
}
func (self BatchResponse) ResultId() *pilosa.GUID {
return self.Id
}
func (self BatchResponse) ResultData() interface{} {
return self.Id
}
func init() {
gob.Register(BatchRequest{})
gob.Register(BatchResponse{})
}
type Batcher struct {
ID pilosa.GUID
Cluster *db.Cluster
Hold *hold.Holder
Transport interface {
Send(message *db.Message, host *pilosa.GUID)
}
}
func NewBatcher(id pilosa.GUID) *Batcher {
return &Batcher{ID: id}
}
func (b *Batcher) Batch(database_name, frame, compressed_bitmap string, bitmap_id uint64, slice int, filter uint64) error {
log.Trace("Batch:", "db:", database_name, " frame:", frame, " slice:", slice, " cb:", compressed_bitmap, " bid:", bitmap_id, "f:", filter)
//determine the fragment_id from database/frame/slice
database := b.Cluster.GetOrCreateDatabase(database_name)
oslice := database.GetOrCreateSlice(slice)
//need to find processid and fragment id for that slice
fragment, err := database.GetFragmentForBitmap(oslice, &db.Bitmap{Id: bitmap_id, FrameType: frame, Filter: filter})
if err == nil {
id := pilosa.NewGUID()
batch := db.Message{Data: BatchRequest{Id: &id, Source: &b.ID, Fragment_id: fragment.GetId(), Bitmap_id: bitmap_id, Compressed_bitmap: compressed_bitmap}}
dest_id := fragment.GetProcess().Id()
b.Transport.Send(&batch, &dest_id)
_, err = b.Hold.Get(&id, 60)
}
return err
}

519
core/etcd.go
View file

@ -1,519 +0,0 @@
package core
import (
"errors"
"fmt"
"os"
"runtime/debug"
"sort"
"strconv"
"strings"
"sync"
"time"
log "github.com/cihub/seelog"
"github.com/coreos/go-etcd/etcd"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
const (
DefaultFragmentAllocLockTTL = 14400 * time.Second
)
type TopologyMapper struct {
namespace string
ID pilosa.GUID
Cluster *db.Cluster
ProcessMap *ProcessMap
SupportedFrames []string
FragmentAllocLockTTL time.Duration
EtcdClient interface {
CreateDir(key string, ttl uint64) (*etcd.Response, error)
Get(key string, sort, recursive bool) (*etcd.Response, error)
RawCreate(key string, value string, ttl uint64) (*etcd.RawResponse, error)
Set(key string, value string, ttl uint64) (*etcd.Response, error)
Watch(prefix string, waitIndex uint64, recursive bool, receiver chan *etcd.Response, stop chan bool) (*etcd.Response, error)
}
Index interface {
AddFragment(db string, frame string, slice int, id pilosa.SUUID)
}
}
func NewTopologyMapper(namespace string) *TopologyMapper {
return &TopologyMapper{
namespace: namespace,
FragmentAllocLockTTL: DefaultFragmentAllocLockTTL,
}
}
func (self *TopologyMapper) Setup() {
log.Warn(self.namespace + "/db")
db_path := self.namespace + "/db"
resp, err := self.EtcdClient.Get(db_path, false, true)
if err != nil {
ee, ok := err.(*etcd.EtcdError)
if ok && ee.ErrorCode == 100 { // node does not exist
resp, err = self.EtcdClient.CreateDir(db_path, 0)
if err != nil {
log.Critical(err)
os.Exit(-1)
}
} else {
log.Critical(err)
os.Exit(-1)
}
}
//need to lock the world
for _, node := range flatten(resp.Node) {
err := self.handlenode(node)
if err != nil {
log.Warn(err)
}
}
}
func (self *TopologyMapper) Run() {
receiver := make(chan *etcd.Response)
go func() {
// TODO: add some terminating measure
// TODO: use modindex to make sure watch catches everything
for {
ns := self.namespace + "/db"
log.Warn(" ETCD watcher:", ns)
stop := make(chan bool)
resp, err := self.EtcdClient.Watch(ns, 0, true, receiver, stop)
log.Warn("TopologyMapper ETCD watcher", resp, err)
}
}()
go func() {
for resp := range receiver {
switch resp.Action {
case "set":
self.handlenode(resp.Node)
case "delete":
self.remove_fragment(resp.Node)
}
// TODO: handle deletes
}
}()
}
type Pair struct {
Key string
Value int
}
// A slice of Pairs that implements sort.Interface to sort by Value.
type PairList []Pair
func (p PairList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p PairList) Len() int { return len(p) }
func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }
func getLightestProcess(m map[string]int) (Pair, error) {
l := len(m)
if l == 0 {
return Pair{}, errors.New("No Processes")
}
processlist := make(PairList, l)
i := 0
for k, v := range m {
processlist[i] = Pair{k, v}
i++
}
sort.Sort(processlist)
return processlist[0], nil
}
func (self *TopologyMapper) GetProcessFragmentCounts() map[string]int {
m := make(map[string]int)
m[self.ID.String()] = 0 //at least have one process if none created
for k, _ := range self.ProcessMap.nodes {
p := k.String()
m[p] = 0 //at least have one process if none created
}
for _, dbs := range self.Cluster.GetDatabases() {
for _, fsi := range dbs.GetFrameSliceIntersects() {
for _, fragment := range fsi.GetFragments() {
process := fragment.GetProcess().Id().String()
if len(process) > 1 {
i := m[process]
i++
m[process] = i
}
}
}
}
return m
}
func (self *TopologyMapper) MakeFragments(db string, slice_int int) error {
lock_key := fmt.Sprintf("%s/lock/%s-%d", self.namespace, db, slice_int)
response, err := self.EtcdClient.RawCreate(lock_key, "0", uint64(self.FragmentAllocLockTTL.Seconds()))
if err == nil {
if response.StatusCode == 201 { //key created
log.Warn("MakeFragments:", db, slice_int)
m := self.GetProcessFragmentCounts()
p, err := getLightestProcess(m)
if err != nil {
log.Warn("MakeFragments: error finding process", db, slice_int, err)
return err
}
// PUT -d "value=5cb315c3-6e1d-4218-89b7-943d1dba985b" http://etcd0:4001/v2/keys/pilosa/0/db/29/frame/d/slice/5/fragment/a2b632fc4001b817/proces
for _, frame := range self.SupportedFrames {
err := self.AllocateFragment(p.Key, db, frame, slice_int)
if err != nil {
log.Warn(err)
}
}
}
}
return nil
}
func (self *TopologyMapper) AllocateFragment(process_guid, db, frame string, slice_int int) error {
//get Lock to create the fragment
//figure out least loaded process..possibly check max process
//to create the node, just write off the items to etcd and the watch should spawn
//be nice if something would notify perhaps queue
//so i need db, frame, slice , fragment_id
fuid := pilosa.NewSUUID().String()
fragment_key := fmt.Sprintf("%s/db/%s/frame/%s/slice/%d/fragment/%s/process", self.namespace, db, frame, slice_int, fuid)
// need to check value to see how many we have left
log.Warn("ALLOC:", process_guid, len(process_guid))
if len(process_guid) > 1 {
_, err := self.EtcdClient.Set(fragment_key, process_guid, 0)
if err != nil {
return err
}
log.Warn("Fragment sent to etcd:", fragment_key, process_guid)
}
return nil
}
func (self *TopologyMapper) handlenode(node *etcd.Node) error {
key := node.Key[len(self.namespace)+1:]
bits := strings.Split(key, "/")
var database *db.Database
var frame *db.Frame
var fragment *db.Fragment
var fragment_id pilosa.SUUID
var slice *db.Slice
var slice_int int
var process_uuid pilosa.GUID
var process *db.Process
var err error
if len(bits) > 8 {
if bits[8] != "process" {
return errors.New("no process")
}
process_uuid, err = pilosa.ParseGUID(node.Value)
if err != nil {
log.Warn("Bad Process Guid", key)
return errors.New("No Process Id")
}
} else {
return err
}
if len(bits) <= 1 || bits[0] != "db" {
return nil
}
if len(bits) > 1 {
database = self.Cluster.GetOrCreateDatabase(bits[1])
}
if len(bits) > 2 {
if bits[2] != "frame" {
return errors.New("no frame")
}
}
if len(bits) > 3 {
frame = database.GetOrCreateFrame(bits[3])
}
if len(bits) > 4 {
if bits[4] != "slice" {
return errors.New("no slice")
}
}
if len(bits) > 5 {
slice_int, err = strconv.Atoi(bits[5])
if err != nil {
return err
}
slice = database.GetOrCreateSlice(slice_int)
}
if len(bits) > 6 {
if bits[6] != "fragment" {
return errors.New("no fragment")
}
}
if len(bits) > 7 {
fragment_id = pilosa.ParseSUUID(bits[7])
fragment = database.GetOrCreateFragment(frame, slice, fragment_id)
}
if len(bits) > 8 {
if bits[8] != "process" {
return errors.New("no process")
}
if err != nil {
log.Warn("Bad UUID:", process_uuid, key)
return err
}
process = db.NewProcess(&process_uuid)
fragment.SetProcess(process)
if self.ID.Equals(&process_uuid) {
self.Index.AddFragment(bits[1], bits[3], slice_int, fragment_id)
}
}
return err
}
func (self *TopologyMapper) remove_fragment(node *etcd.Node) error {
log.Warn(" hot remove_fragment (Not Supported yet):", node)
return nil
}
func flatten(node *etcd.Node) []*etcd.Node {
nodes := []*etcd.Node{node}
for i := 0; i < len(node.Nodes); i++ {
nodes = append(nodes, flatten(&node.Nodes[i])...)
}
return nodes
}
type Node struct {
id *pilosa.GUID
ip string
port_tcp int
port_http int
}
type ProcessMap struct {
nodes map[pilosa.GUID]*db.Process
mutex sync.Mutex
}
func NewProcessMap() *ProcessMap {
p := ProcessMap{}
p.nodes = make(map[pilosa.GUID]*db.Process)
return &p
}
func (self *ProcessMap) AddProcess(process *db.Process) {
self.mutex.Lock()
defer self.mutex.Unlock()
self.nodes[process.Id()] = process
}
func (self *ProcessMap) GetProcess(id *pilosa.GUID) (*db.Process, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
if id == nil {
debug.PrintStack()
return nil, errors.New("Nil process")
}
process, ok := self.nodes[*id]
if !ok {
return nil, errors.New("No such process")
}
return process, nil
}
func (self *ProcessMap) GetOrAddProcess(id *pilosa.GUID) *db.Process {
process, err := self.GetProcess(id)
if err != nil {
process = db.NewProcess(id)
self.AddProcess(process)
}
return process
}
func (self *ProcessMap) GetHost(id *pilosa.GUID) (string, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
process, ok := self.nodes[*id]
if !ok {
return "", errors.New("Process does not exist")
}
return process.Host(), nil
}
func (self *ProcessMap) GetPortTcp(id *pilosa.GUID) (int, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
process, ok := self.nodes[*id]
if !ok {
return 0, errors.New("Process does not exist")
}
return process.PortTcp(), nil
}
func (self *ProcessMap) GetPortHttp(id *pilosa.GUID) (int, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
process, ok := self.nodes[*id]
if !ok {
return 0, errors.New("Process does not exist")
}
return process.PortHttp(), nil
}
func (self *ProcessMap) GetMetadata() map[string]map[string]interface{} {
self.mutex.Lock()
defer self.mutex.Unlock()
out := make(map[string]map[string]interface{})
for id, process := range self.nodes {
pdata := make(map[string]interface{})
pdata["host"] = process.Host()
pdata["port_tcp"] = process.PortTcp()
pdata["port_http"] = process.PortHttp()
out[id.String()] = pdata
}
return out
}
type ProcessMapper struct {
receiver chan etcd.Response
commands chan ProcessMapperCommand
namespace string
ID pilosa.GUID
ProcessMap *ProcessMap
TCPPort int
HTTPPort int
Host string
EtcdClient interface {
Get(key string, sort, recursive bool) (*etcd.Response, error)
Set(key string, value string, ttl uint64) (*etcd.Response, error)
Watch(prefix string, waitIndex uint64, recursive bool, receiver chan *etcd.Response, stop chan bool) (*etcd.Response, error)
}
}
func NewProcessMapper(namespace string) *ProcessMapper {
return &ProcessMapper{
receiver: make(chan etcd.Response),
commands: make(chan ProcessMapperCommand),
namespace: namespace,
}
}
type ProcessMapperCommand struct {
key string
}
func getKey(input string) string {
bits := strings.Split(input, "/")
return bits[len(bits)-1]
}
func (self *ProcessMapper) getnode(u *pilosa.GUID) *Node {
return new(Node)
}
func crash_on_error(err error) {
if err != nil {
log.Critical(err)
os.Exit(-1)
}
}
func (self *ProcessMapper) handlenode(node *etcd.Node) error {
var err error
var process *db.Process
key := node.Key[len(self.namespace)+1:]
bits := strings.Split(key, "/")
if len(bits) <= 1 || bits[0] != "process" {
return nil
}
if len(bits) >= 2 {
id_string := bits[1]
id, err := pilosa.ParseGUID(id_string)
if err != nil {
return errors.New("Invalid GUID: " + id_string + " (" + key + ")")
}
process = self.ProcessMap.GetOrAddProcess(&id)
}
if len(bits) >= 3 {
switch bits[2] {
case "port_tcp":
port_tcp, _ := strconv.Atoi(node.Value)
process.SetPortTcp(port_tcp)
case "port_http":
port_http, _ := strconv.Atoi(node.Value)
process.SetPortHttp(port_http)
case "host":
host := node.Value
process.SetHost(host)
}
}
return err
}
func (self *ProcessMapper) Run() {
path := self.namespace + "/process"
self_path := path + "/" + self.ID.String()
log.Warn("Writing configuration to etcd...")
log.Warn(self_path)
var err error
_, err = self.EtcdClient.Set(self_path+"/port_tcp", strconv.Itoa(self.TCPPort), 0)
crash_on_error(err)
_, err = self.EtcdClient.Set(self_path+"/port_http", strconv.Itoa(self.HTTPPort), 0)
crash_on_error(err)
_, err = self.EtcdClient.Set(self_path+"/host", self.Host, 0)
crash_on_error(err)
response, err := self.EtcdClient.Get(path, false, true)
for _, node := range flatten(response.Node) {
err := self.handlenode(node)
if err != nil {
out := spew.Sdump(node)
log.Warn(err, out)
}
}
receiver := make(chan *etcd.Response)
stop := make(chan bool)
go func() {
// TODO: error check and restart watcher
// TODO: use modindex to make sure watch catches everything
_, _ = self.EtcdClient.Watch(path, 0, true, receiver, stop)
}()
go func() {
for response = range receiver {
switch response.Action {
case "set":
self.handlenode(response.Node)
}
// TODO: handle deletes
}
}()
}

854
core/http.go
View file

@ -1,854 +0,0 @@
package core
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/gob"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httputil"
"os"
"reflect"
"runtime"
"strconv"
"strings"
"time"
notify "github.com/bitly/go-notify"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/gorilla/websocket"
"github.com/kr/s3/s3util"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/statsd"
)
const DefaultRequestLogPath = "/tmp/set_bit_log"
var RequestLogPath = DefaultRequestLogPath
type WebService struct {
end chan bool
ID pilosa.GUID
Version string
Port int
DefaultDB string
SetBitLogEnabled bool
Cluster *db.Cluster
ProcessMap *ProcessMap
Batcher *Batcher
Executor interface {
RunPQL(database_name string, pql string) (interface{}, error)
}
Pinger interface {
Ping(process_id *pilosa.GUID) (*time.Duration, error)
}
TopologyMapper interface {
MakeFragments(db string, slice_int int) error
}
Transport interface {
Push(message *db.Message)
}
}
func NewWebService() *WebService {
return &WebService{
end: make(chan bool),
}
}
type Flusher struct {
flusher chan bool
}
func (self *Flusher) Handle(w http.ResponseWriter, r *http.Request) {
self.flusher <- true
}
func NewFlusher(flusher chan bool) http.HandlerFunc {
f := Flusher{flusher}
return f.Handle
}
type RequestLogger struct {
handler http.HandlerFunc
logger chan []byte
}
func NewRequestLogger(handler http.HandlerFunc, logger chan []byte) http.HandlerFunc {
r := RequestLogger{handler, logger}
return r.Handle
}
func (self *RequestLogger) Handle(w http.ResponseWriter, r *http.Request) {
// Grab a dump of the incoming request
dump, err := httputil.DumpRequest(r, true /*dump the body also*/)
if err != nil {
log.Warn("Dump Failure", err)
}
self.handler(w, r)
self.logger <- dump
}
type LogRecord struct {
When time.Time
Data_x64 string
}
func NewLogRecord(t time.Time, data []byte) LogRecord {
var b bytes.Buffer
w := gzip.NewWriter(&b)
w.Write(data)
w.Flush()
w.Close()
e := base64.StdEncoding.EncodeToString(b.Bytes())
x := LogRecord{t, e}
return x
}
func genFileName(id string) string {
//bucket/YYYY/MM/DDHHMMSS.id.log
t := time.Now()
//base := "http://pilosa.umbel.com.s3.amazonaws.com/bit_log"
return fmt.Sprintf("%s%s.%s.log", RequestLogPath, t.Format("/2006/01/02/15/04-05"), id)
}
func flush(requests []LogRecord, id string, records_to_dump int) {
dest := genFileName(id)
w, err := createFile(dest)
if err != nil {
log.Warn("Error opening outfile ", dest)
log.Warn(err)
return
}
defer w.Close()
encoder := json.NewEncoder(w)
for i := 0; i < records_to_dump; i++ {
encoder.Encode(requests[i])
}
}
func Logger(in chan []byte, end chan bool, id string, flusher chan bool) {
var buffer = make([]LogRecord, 2048, 2048)
i := 0
for {
select {
case raw := <-in:
logRecord := NewLogRecord(time.Now(), raw)
buffer[i] = logRecord
i += 1
if i > 2047 {
flush(buffer, id, i)
i = 0
}
case <-flusher:
if i > 0 {
flush(buffer, id, i)
i = 0
}
case <-end:
flush(buffer, id, i)
log.Info("Shutdown Logger")
return
}
}
}
func (self *WebService) Run() {
port_string := strconv.Itoa(self.Port)
log.Info("Serving HTTP on port:", port_string)
logger_chan := make(chan []byte, 1024)
flusher := make(chan bool)
mux := http.NewServeMux()
mux.HandleFunc("/message", self.HandleMessage)
mux.HandleFunc("/query", self.HandleQuery)
mux.HandleFunc("/stats", self.HandleStats)
mux.HandleFunc("/status", self.HandleStatus)
mux.HandleFunc("/info", self.HandleInfo)
mux.HandleFunc("/processes", self.HandleProcesses)
mux.HandleFunc("/listen/ws", self.HandleListenWS)
mux.HandleFunc("/listen/stream", self.HandleListenStream)
mux.HandleFunc("/listen", self.HandleListen)
mux.HandleFunc("/test", self.HandleTest)
mux.HandleFunc("/version", self.HandleVersion)
mux.HandleFunc("/ping", self.HandlePing)
mux.HandleFunc("/batch", self.HandleBatch)
mux.HandleFunc("/load", self.HandleLoad)
if self.SetBitLogEnabled {
mux.HandleFunc("/set_bits", NewRequestLogger(self.HandleSetBit, logger_chan))
} else {
mux.HandleFunc("/set_bits", self.HandleSetBit)
}
mux.HandleFunc("/clear_bits", self.HandleClearBit)
//mux.HandleFunc("/set_bits", self.HandleSetBit)
mux.HandleFunc("/flush", NewFlusher(flusher))
s := &http.Server{
Addr: ":" + port_string,
Handler: mux,
}
go Logger(logger_chan, self.end, self.ID.String(), flusher)
s.ListenAndServe()
}
func (self *WebService) Shutdown() {
self.end <- true
}
func (self *WebService) HandleMessage(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
var message db.Message
decoder := json.NewDecoder(r.Body)
if decoder.Decode(&message) != nil {
http.Error(w, "Invalid JSON", http.StatusBadRequest)
return
}
}
func (self *WebService) HandleLoad(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
decoder := json.NewDecoder(r.Body)
var obj JsonObject
err := decoder.Decode(&obj)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
database_name, ok := obj["db"]
if !ok {
http.Error(w, "Provide a database (db)", http.StatusNotFound)
return
}
db := database_name.(string)
ms_, ok := obj["max_slice"]
if ok {
ms := int(ms_.(float64))
database := self.Cluster.GetOrCreateDatabase(db)
ns, _ := database.NumSlices()
if ns <= ms {
for i := ns; i <= ms; i++ {
log.Info("Load Create Slice ", i)
self.TopologyMapper.MakeFragments(db, i)
}
http.Error(w, "Needed Slices", http.StatusNotFound)
return
}
}
_, ok = obj["id"]
if !ok {
http.Error(w, "Provide a bitmap id (id)", http.StatusNotFound)
return
}
t := float64(obj["id"].(float64))
bitmap_id := uint64(t)
frame, ok := obj["frame"]
if !ok {
http.Error(w, "Provide a frame (frame)", http.StatusNotFound)
return
}
api_string, ok := obj["bitmap"]
if !ok {
http.Error(w, "Provide a compressed base64 bitmap (bitmap)", http.StatusNotFound)
return
}
_, ok = obj["filter"]
if !ok {
http.Error(w, "Provide a filter for categories", http.StatusNotFound)
return
}
t = float64(obj["filter"].(float64))
filter := uint64(t)
results := FromApiString(self.Batcher, database_name.(string), frame.(string), api_string.(string), bitmap_id, filter)
encoder := json.NewEncoder(w)
err = encoder.Encode(results)
if err != nil {
log.Warn("Error Load results")
log.Warn(spew.Sdump(r.Form))
err = encoder.Encode("Bad Batch Request")
}
}
func (self *WebService) HandleBatch(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
err := r.ParseForm()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
database_name := r.Form.Get("db")
if database_name == "" {
http.Error(w, "Provide a database (db)", http.StatusNotFound)
return
}
bitmap_id_string := r.Form.Get("id")
if bitmap_id_string == "" {
http.Error(w, "Provide a bitmap id (id)", http.StatusNotFound)
return
}
bitmap_id, _ := strconv.ParseUint(bitmap_id_string, 10, 64)
slice_string := r.Form.Get("slice")
if bitmap_id_string == "" {
http.Error(w, "Provide a slice (slice)", http.StatusNotFound)
return
}
slice, _ := strconv.ParseInt(slice_string, 10, 32)
frame := r.Form.Get("frame")
if bitmap_id_string == "" {
http.Error(w, "Provide a frame (frame)", http.StatusNotFound)
return
}
compressed_bitmap := r.Form.Get("bitmap")
if bitmap_id_string == "" {
http.Error(w, "Provide a compressed base64 bitmap (bitmap)", http.StatusNotFound)
return
}
filter_string := r.Form.Get("filter")
filter, _ := strconv.ParseInt(filter_string, 10, 32)
if bitmap_id_string == "" {
http.Error(w, "Provide a filter for categories", http.StatusNotFound)
return
}
results := self.Batcher.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice), uint64(filter))
encoder := json.NewEncoder(w)
err = encoder.Encode(results)
if err != nil {
log.Warn("Error Batch results")
log.Warn(spew.Sdump(r.Form))
err = encoder.Encode("Bad Batch Request")
}
}
func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
statsd.SendInc("webservice_Query")
err := r.ParseForm()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
database_name := r.Form.Get("db")
if database_name == "" {
database_name = self.DefaultDB
}
if database_name == "" {
http.Error(w, "Provide a database (db)", http.StatusNotFound)
return
}
if !self.Cluster.IsValidDatabase(database_name) {
http.Error(w, "Unknown Database:"+database_name, http.StatusNotFound)
return
}
pql := r.Form.Get("pql")
if pql == "" {
http.Error(w, "Provide a valid query string (pql)", http.StatusNotFound)
return
}
_, bits := r.Form["bits"]
log.Debug("PQL:", database_name, pql)
results, err := self.Executor.RunPQL(database_name, pql)
if err != nil {
log.Warn("PQL Exec Error:", err.Error(), database_name, pql)
http.Error(w, "Error encoding: "+err.Error(), http.StatusInternalServerError)
return
}
switch r := results.(type) { //a hack to handle empty sets
case []pilosa.Pair:
if len(r) == 0 {
results = []int{}
}
case []byte: //can i figure out the type of the compressed string here?
if bits {
reader, _ := gzip.NewReader(bytes.NewReader(r))
b, _ := ioutil.ReadAll(reader)
result := pilosa.NewBitmap()
result.FromBytes(b)
results = result.Bits()
}
}
if results == nil {
log.Warn("Empty results:", database_name, pql)
http.Error(w, "Error encoding: "+err.Error(), http.StatusInternalServerError)
return
}
if err != nil {
http.Error(w, "Error running query: "+err.Error(), http.StatusInternalServerError)
return
}
encoder := json.NewEncoder(w)
err = encoder.Encode(results)
if err != nil {
log.Warn("Encode Error :", database_name, pql, err.Error())
return
}
}
type JsonObject map[string]interface{}
// post this json to the body
//the frame type with extension ".t" are handled a bit differnt
// it generats the timestamp based bitmap_ids
//[{ "db": "3", "frame":"b.n","profile_id": 122,"filter":0, "bitmap_id":123},
// { "db": "3", "frame":"b.n","profile_id": 122,"filter":2, "bitmap_id":124},
// { "db": "3", "frame":"t.t","profile_id": 122,"filter":2, "bitmap_id":124, "timestamp":"2014-04-03 13:01:04"}]'
//
func bitmaps(frame string, obj JsonObject) chan uint64 {
c := make(chan uint64)
go func() {
const shortFormT = "2006-01-02T15:04:05"
const shortFormS = "2006-01-02 15:04:05"
t := float64(obj["bitmap_id"].(float64))
base_id := uint64(t)
if strings.HasSuffix(frame, ".t") {
timestamp, present := obj["timestamp"].(string)
if !present || timestamp == "2014-01-01 00:00:00" { //skip the default timestamp
c <- base_id
} else {
quantum := pilosa.YMDH
if val, ok := obj["time_granularity"]; ok {
switch val {
case "Y":
quantum = pilosa.Y
case "M":
quantum = pilosa.YM
case "D":
quantum = pilosa.YMD
}
}
shortForm := shortFormS
if strings.Contains(timestamp, "T") {
shortForm = shortFormT
}
atime, _ := time.Parse(shortForm, timestamp)
for i, id := range pilosa.GetTimeIds(base_id, atime, quantum) {
c <- id
if i > 10 {
log.Warn("TO MANY TIMEIDS", base_id, atime, quantum)
break
}
}
}
} else {
c <- base_id
}
close(c)
}()
return c
}
type SBResult struct {
Bitmap_id uint64
Frame string
Filter uint64
Profile_id uint64
Result interface{}
}
func init() {
gob.Register(SBResult{})
}
func (self *WebService) HandleClearBit(w http.ResponseWriter, r *http.Request) {
self.HandleBit(w, r, false)
}
func (self *WebService) HandleSetBit(w http.ResponseWriter, r *http.Request) {
self.HandleBit(w, r, true)
}
func (self *WebService) HandleBit(w http.ResponseWriter, r *http.Request, ToSet bool) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
decoder := json.NewDecoder(r.Body)
var args []JsonObject
err := decoder.Decode(&args)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
//[{ "db": "3", "frame":"brand.","profile_id": 122,"filter":0, "bitmap_id":123}]
var results []SBResult
if len(args) > 4096 {
log.Warn("Request too large:", len(args))
http.Error(w, "Request To large", http.StatusBadRequest)
return
}
//remoteSetBit := NewRemoteSetBit()
//remoteSetBit.ID = self.ID
//remoteSetBit.ProcessMap = self.ProcessMap
//remoteSetBit.Hold = self.Hold
//remoteSetBit.Transport = self.Transport
for _, obj := range args {
if obj["profile_id"] == nil {
http.Error(w, "Missing Profile", http.StatusBadRequest)
return
}
t := float64(obj["profile_id"].(float64))
profile_id := uint64(t)
if obj["db"] == nil {
http.Error(w, "Missing db", http.StatusBadRequest)
return
}
dbs := obj["db"].(string)
if obj["frame"] == nil {
http.Error(w, "Missing Frame", http.StatusBadRequest)
return
}
frame := obj["frame"].(string)
if obj["filter"] == nil {
http.Error(w, "Missing Filter", http.StatusBadRequest)
return
}
t = float64(obj["filter"].(float64))
filter := uint64(t)
for bitmap_id := range bitmaps(frame, obj) {
var pql string
if ToSet {
pql = fmt.Sprintf("set(%d, %s, %d, %d)", bitmap_id, frame, filter, profile_id)
} else {
pql = fmt.Sprintf("clear(%d, %s, %d, %d)", bitmap_id, frame, filter, profile_id)
}
result, err := self.Executor.RunPQL(dbs, pql)
bundle := SBResult{bitmap_id, frame, filter, profile_id, result}
results = append(results, bundle)
if err != nil {
log.Warn("Error running set_bit", dbs, frame, profile_id, ToSet)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
}
encoder := json.NewEncoder(w)
err = encoder.Encode(results)
if err != nil {
log.Warn("JSON SetBit ERROR:", err, ToSet)
return
}
}
func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
encoder := json.NewEncoder(w)
m := &runtime.MemStats{}
runtime.ReadMemStats(m)
err := encoder.Encode(m)
if err != nil {
log.Warn("Error encoding stats")
http.Error(w, "Error econding stats", http.StatusMethodNotAllowed)
}
}
func (self *WebService) HandleInfo(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
spew.Fdump(w, self.Cluster)
}
func (self *WebService) HandleVersion(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
fmt.Fprintf(w, "Pilosa v.("+self.Version+")\n")
}
func (self *WebService) HandleTest(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
spew.Dump("TEST!")
msg := new(db.Message)
msg.Data = "mystring"
self.Transport.Push(msg)
msg2 := new(db.Message)
msg2.Data = 789
self.Transport.Push(msg2)
}
func (self *WebService) HandleProcesses(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
encoder := json.NewEncoder(w)
processes := self.ProcessMap.GetMetadata()
err := encoder.Encode(processes)
if err != nil {
http.Error(w, "Error Encoding", http.StatusBadRequest)
}
}
func (self *WebService) HandlePing(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
err := r.ParseForm()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
process_string := r.Form.Get("process")
process_id, err := pilosa.ParseGUID(process_string)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
_, err = self.ProcessMap.GetProcess(&process_id)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
duration, err := self.Pinger.Ping(&process_id)
if err != nil {
spew.Fdump(w, err)
return
}
encoder := json.NewEncoder(w)
encoder.Encode(map[string]float64{"duration": duration.Seconds()})
}
func (self *WebService) HandleListen(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`
<html><head><title>Pilosa - streaming client</title></head><body>
<style type="text/css">
dt.inbox { background-color: #efe; }
dt.outbox { background-color: #eef; }
</style>
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script type="text/javascript">
$dl = $("<dl/>")
$dl.on('click', 'dt', function(e) {
$(this).next().toggle()
})
$("body").append($dl)
ws = new WebSocket("ws://" + window.location.host + "/listen/ws")
ws.onmessage = function(mes) {
obj = JSON.parse(mes.data)
if (obj.host) {
$dl.append('<dt class="outbox">&rarr;' + obj.type + ' (to: ' + obj.host + ')</dt>')
} else {
$dl.append('<dt class="inbox">&larr;' + obj.type + '</dt>')
}
$dl.append('<dd style="display:none;"><pre>' + obj.dump + obj.host + '</pre></dd>')
}
</script>
</body></html>
`))
}
func (self *WebService) streamer(writer func(map[string]interface{}) error) {
inbox := make(chan interface{}, 10)
outbox := make(chan interface{}, 10)
notify.Start("inbox", inbox)
notify.Start("outbox", outbox)
var obj interface{}
var data interface{}
var inmessage *db.Message
var outmessage *db.Envelope
var host string
for {
select {
case obj = <-outbox:
outmessage = obj.(*db.Envelope)
data = outmessage.Message.Data
host = outmessage.Host.String()
case obj = <-inbox:
inmessage = obj.(*db.Message)
data = inmessage.Data
host = ""
}
typ := reflect.TypeOf(data)
err := writer(map[string]interface{}{
"type": typ.String(),
"dump": spew.Sdump(data),
"host": host,
})
if err != nil {
log.Info("stopping")
notify.Stop("inbox", inbox)
notify.Stop("outbox", outbox)
drain(inbox)
drain(outbox)
return
}
}
}
func (self *WebService) HandleListenWS(w http.ResponseWriter, r *http.Request) {
defer func() {
err := recover()
out := spew.Sdump(err)
log.Info(out)
}()
ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
if _, ok := err.(websocket.HandshakeError); ok {
http.Error(w, "Not a websocket handshake", 400)
return
} else if err != nil {
log.Warn(err)
return
}
self.streamer(func(data map[string]interface{}) error {
return ws.WriteJSON(data)
})
}
func drain(ch chan interface{}) {
for {
switch {
case <-ch:
default:
return
}
}
}
func (self *WebService) HandleListenStream(w http.ResponseWriter, r *http.Request) {
defer func() {
err := recover()
spew.Dump(err)
}()
writer := json.NewEncoder(w)
flusher := w.(http.Flusher)
self.streamer(func(data map[string]interface{}) error {
err := writer.Encode(data)
flusher.Flush()
return err
})
}
func (self *WebService) HandleStatus(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`
<html><head><title>Pilosa - status</title></head><body>
<style type="text/css">
td {
background: #eee;
}
</style>
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script type="text/javascript">
$table = $("<table><tr><th>process id</th><th>host</th><th>tcp port</th><th>http port</th><th>latency</th></tr></table>")
$("body").append($table)
$.ajax('/processes', {
dataType: "json",
success: function(resp) {
$.each(resp, function(index, value) {
$table.append('<tr><td>' + index + '</td><td>' + value.host + '</td><td>' + value.port_tcp + '</td><td>' + value.port_http + '</td><td><button class="pinger"/></td></tr>')
})
}
})
$table.on('click', 'button.pinger', function(e) {
var $td = $(this).parent()
var $tr = $td.parent()
var id = $tr.children('td').eq(0).text()
$.ajax('/ping?process=' + id, {
dataType: 'json',
success: function(resp) {
$td.html(resp.duration)
}
})
})
</script>
</body></html>
`))
}
func createFile(s string) (io.WriteCloser, error) {
if isURL(s) {
return s3util.Create(s, nil, nil)
}
return os.Create(s)
}
func isURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
}

95
core/load.go
View file

@ -1,95 +0,0 @@
package core
// #cgo CFLAGS:-mpopcnt
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/binary"
"errors"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
)
func copy_raw(src [32]uint64) pilosa.Blocks {
o := make(pilosa.Blocks, 32)
for k, v := range src {
o[k] = v
}
return o
}
func sendBitmap(batcher *Batcher, bitmap *pilosa.Bitmap, db string, frame string, bitmap_id, filter uint64, slice int, finish chan error) {
if slice < 0 {
log.Warn("Bad split", db, frame, slice, bitmap_id)
finish <- errors.New("BadSplit")
return
}
// Compress bitmap.
var b bytes.Buffer
w := gzip.NewWriter(&b)
w.Write(bitmap.ToBytes())
w.Flush()
w.Close()
buf := base64.StdEncoding.EncodeToString(b.Bytes())
results := batcher.Batch(db, frame, buf, bitmap_id, slice, filter)
finish <- results
}
func FromApiString(batcher *Batcher, db string, frame string, api_string string, bitmap_id, filter uint64) string {
compressed_data, err := base64.StdEncoding.DecodeString(api_string)
if err != nil {
log.Warn(err)
return "Bad"
}
reader, err := gzip.NewReader(bytes.NewReader(compressed_data))
if err != nil {
log.Warn(err)
return "Bad"
}
var numChunks uint64
err = binary.Read(reader, binary.LittleEndian, &numChunks)
if err != nil {
log.Warn(err)
return "Bad"
}
first := true
bitmap := pilosa.NewBitmap()
last_slice := pilosa.CounterMask
sent_count := 0
finish := make(chan error)
for i := uint64(0); i < numChunks; i++ {
var raw struct {
Key uint64
Block [32]uint64
}
binary.Read(reader, binary.LittleEndian, &raw)
slice := raw.Key >> 5
if slice != last_slice {
if first {
first = false
} else {
//make async later
sent_count += 1
go sendBitmap(batcher, bitmap, db, frame, bitmap_id, filter, int(last_slice), finish)
bitmap = pilosa.NewBitmap()
}
last_slice = slice
}
o := copy_raw(raw.Block)
chunk := &pilosa.Chunk{Key: raw.Key, Value: o}
bitmap.AddChunk(chunk)
}
sent_count += 1
go sendBitmap(batcher, bitmap, db, frame, bitmap_id, filter, int(last_slice), finish)
for i := 0; i < sent_count; i++ {
<-finish
}
return "OK"
}

62
core/ping.go
View file

@ -1,62 +0,0 @@
package core
import (
"encoding/gob"
"time"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
type PingRequest struct {
Id *pilosa.GUID
Source *pilosa.GUID
}
type PongRequest struct {
Id *pilosa.GUID
}
func (self PongRequest) ResultId() *pilosa.GUID {
return self.Id
}
func (self PongRequest) ResultData() interface{} {
return self.Id
}
func init() {
gob.Register(PingRequest{})
gob.Register(PongRequest{})
}
type Pinger struct {
ID pilosa.GUID
Hold interface {
Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error)
}
Transport interface {
Send(message *db.Message, host *pilosa.GUID)
}
}
func NewPinger(id pilosa.GUID) *Pinger {
return &Pinger{
ID: id,
}
}
func (self *Pinger) Ping(process_id *pilosa.GUID) (*time.Duration, error) {
id := pilosa.NewGUID()
ping := db.Message{Data: PingRequest{Id: &id, Source: &self.ID}}
start := time.Now()
self.Transport.Send(&ping, process_id)
_, err := self.Hold.Get(&id, 60*time.Second)
if err != nil {
return nil, err
}
end := time.Now()
dur := end.Sub(start)
return &dur, nil
}

127
core/remotebits.go
View file

@ -1,127 +0,0 @@
package core
import (
"encoding/gob"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
type RemoteSetBit struct {
requests []remote_task
cluster map[*pilosa.GUID][]BitmapRequestItem
ID pilosa.GUID
ProcessMap *ProcessMap
Hold interface {
Get(id *pilosa.GUID, timeout int) (interface{}, error)
}
Transport interface {
Send(message *db.Message, host *pilosa.GUID)
}
}
func init() {
gob.Register(BitmapRequestItem{})
gob.Register(BitsRequest{})
gob.Register(BitsResponse{})
}
type BitsRequest struct {
Bits []BitmapRequestItem
ReturnProcessId pilosa.GUID
QueryId pilosa.GUID
DestProcessId pilosa.GUID
}
type BitmapRequestItem struct {
Fragment_id pilosa.SUUID
Bitmap_id uint64
Profile_id uint64
Filter uint64
Frame string
SetUnset bool
}
func NewRemoteSetBit() *RemoteSetBit {
obj := new(RemoteSetBit)
obj.cluster = make(map[*pilosa.GUID][]BitmapRequestItem)
return obj
}
func (self *RemoteSetBit) Request() {
self.requests = make([]remote_task, 0)
source_process, _ := self.ProcessMap.GetProcess(&self.ID)
for process, request := range self.cluster {
random_id := pilosa.NewGUID()
msg := new(db.Message)
msg.Data = BitsRequest{
Bits: request,
ReturnProcessId: source_process.Id(),
QueryId: random_id,
DestProcessId: *process,
}
wait := len(request) * 10
if wait < 10 {
wait = 10
}
self.requests = append(self.requests, remote_task{random_id, wait})
self.Transport.Send(msg, process)
}
}
type remote_task struct {
id pilosa.GUID
wait_time int
}
func (self *RemoteSetBit) MergeResults(local_results []SBResult) []SBResult {
answers := make(chan []SBResult)
for _, task := range self.requests {
go func(task remote_task) {
value, err := self.Hold.Get(&task.id, task.wait_time) //eiher need to be the frame process or the handler process?
if value == nil {
log.Warn("Bad RemoteSetBit Result:", err)
empty := make([]SBResult, 0, 0)
answers <- empty
} else {
answers <- value.([]SBResult)
}
}(task)
}
for i := 0; i < len(self.requests); i++ {
batch := <-answers
for _, item := range batch {
local_results = append(local_results, item)
}
}
close(answers)
return local_results
}
func (self *RemoteSetBit) Add(frag *db.Fragment, bitmap_id, profile_id, filter uint64, frame string, SetUnset bool) {
x, found := self.cluster[frag.GetProcessId()]
if !found {
x = make([]BitmapRequestItem, 0)
}
x = append(x, BitmapRequestItem{frag.GetId(), bitmap_id, profile_id, filter, frame, SetUnset})
self.cluster[frag.GetProcessId()] = x
}
type BitsResponse struct {
Id *pilosa.GUID
Items []SBResult
}
func (self *BitsResponse) ResultId() *pilosa.GUID {
return self.Id
}
func (self *BitsResponse) ResultData() interface{} {
return self.Items
}

37
core/stopper.go
View file

@ -1,37 +0,0 @@
package core
import (
"sync"
)
type Stopper struct {
TermChans []chan int
DoneChans []chan int
Mutex sync.RWMutex
}
func (stopper *Stopper) Stop() {
var i chan int
var o chan int
stopper.Mutex.RLock()
for _, i = range stopper.TermChans {
go func(i chan int) {
i <- 1
}(i)
}
for _, o = range stopper.DoneChans {
<-o
}
stopper.Mutex.RUnlock()
return
}
func (stopper *Stopper) GetExitChannels() (chan int, chan int) {
termchan := make(chan int, 1)
donechan := make(chan int, 1)
stopper.Mutex.Lock()
stopper.TermChans = append(stopper.TermChans, termchan)
stopper.DoneChans = append(stopper.DoneChans, donechan)
stopper.Mutex.Unlock()
return termchan, donechan
}

3
db/constants.go
View file

@ -1,3 +0,0 @@
package db
const SLICE_WIDTH = 65536

25
db/db.go
View file

@ -1,25 +0,0 @@
package db
import (
"encoding/gob"
"github.com/umbel/pilosa"
)
type Message struct {
Data interface{} `json:"data"`
}
type Envelope struct {
Message *Message
Host *pilosa.GUID
}
type HoldResult interface {
ResultId() *pilosa.GUID
ResultData() interface{}
}
func init() {
gob.Register(Message{})
}

446
db/topology.go
View file

@ -1,446 +0,0 @@
package db
import (
"errors"
"fmt"
"sync"
log "github.com/cihub/seelog"
"github.com/stathat/consistent"
"github.com/umbel/pilosa"
)
// SupportedFrames is a list of frame types that are supported.
var SupportedFrames = []string{"default"}
var FrameDoesNotExistError = errors.New("Frame does not exist.")
var InvalidFrameError = errors.New("Invalid frame.")
var SliceDoesNotExistError = errors.New("Slice does not exist.")
var FragmentDoesNotExistError = errors.New("Fragment does not exist.")
var FrameSliceIntersectDoesNotExistError = errors.New("FrameSliceIntersect does not exist.")
type Location struct {
ProcessId *pilosa.GUID
FragmentId pilosa.SUUID
}
type Process struct {
id *pilosa.GUID
host string
port_tcp int
port_http int
mutex sync.Mutex
}
func NewProcess(id *pilosa.GUID) *Process {
return &Process{id: id}
}
func (self *Process) Id() pilosa.GUID {
self.mutex.Lock()
defer self.mutex.Unlock()
return *self.id
}
func (self *Process) Host() string {
self.mutex.Lock()
defer self.mutex.Unlock()
return self.host
}
func (self *Process) SetHost(host string) {
self.mutex.Lock()
defer self.mutex.Unlock()
self.host = host
}
func (self *Process) PortTcp() int {
self.mutex.Lock()
defer self.mutex.Unlock()
return self.port_tcp
}
func (self *Process) SetPortTcp(port int) {
self.mutex.Lock()
defer self.mutex.Unlock()
self.port_tcp = port
}
func (self *Process) PortHttp() int {
self.mutex.Lock()
defer self.mutex.Unlock()
return self.port_http
}
func (self *Process) SetPortHttp(port int) {
self.mutex.Lock()
defer self.mutex.Unlock()
self.port_http = port
}
/////////// CLUSTERS
//////////////////////////////////////////////////////////////////////
// Represents the entire cluster, and a reference to the Node this instance is
// running on
type Cluster struct {
databases map[string]*Database
mutex sync.Mutex
}
func NewCluster() *Cluster {
cluster := Cluster{}
cluster.databases = make(map[string]*Database)
return &cluster
}
func (self *Cluster) GetDatabases() map[string]*Database {
return self.databases
}
func (self *Cluster) IsValidDatabase(dbname string) bool {
for name, _ := range self.GetDatabases() {
if name == dbname {
return true
}
}
return false
}
/////////// DATABASES
//////////////////////////////////////////////////////////////////////
// A database is a collection of all the frames within a given profile space
type Database struct {
Name string
frames []*Frame
slices []*Slice
frame_slice_intersects []*FrameSliceIntersect
mutex sync.Mutex
}
func (self *Database) GetFrameSliceIntersects() []*FrameSliceIntersect {
return self.frame_slice_intersects
}
// Add a database to a cluster
func (c *Cluster) addDatabase(name string) *Database {
database := Database{Name: name}
if c.databases == nil {
c.databases = make(map[string]*Database)
}
c.databases[name] = &database
return &database
}
func (c *Cluster) getDatabase(name string) (*Database, error) {
value, ok := c.databases[name]
if !ok {
return nil, errors.New("The database does not exist!")
} else {
return value, nil
}
}
func (c *Cluster) GetOrCreateDatabase(name string) *Database {
c.mutex.Lock()
defer c.mutex.Unlock()
database, err := c.getDatabase(name)
if err == nil {
return database
}
return c.addDatabase(name)
}
func stringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
func (d *Database) IsValidFrame(name string) bool {
return stringInSlice(name, SupportedFrames)
}
// Count the number of slices in a database
func (d *Database) NumSlices() (int, error) {
if len(d.slices) < 1 {
return 0, errors.New("Database is empty")
}
return len(d.slices), nil
}
// return the slice_ids that are in a database
func (d *Database) SliceIds() ([]int, error) {
var rtn []int
for _, slice := range d.slices {
rtn = append(rtn, slice.id)
}
return rtn, nil
}
///////// FRAMES
//////////////////////////////////////////////////////////////////////
// A frame is a collection of slices in a given category
// (brands, demographics, etc), specific to a database
type Frame struct {
name string
}
// Get a frame from a database
func (d *Database) getFrame(name string) (*Frame, error) {
// should we check here for supported frames?
if !d.IsValidFrame(name) {
return nil, InvalidFrameError
}
for _, frame := range d.frames {
if frame.name == name {
return frame, nil
}
}
return nil, FrameDoesNotExistError
}
// Add a frame to a database
func (d *Database) addFrame(name string) *Frame {
frame := Frame{name: name}
d.frames = append(d.frames, &frame)
// add intersections
for _, slice := range d.slices {
d.AddFrameSliceIntersect(&frame, slice)
}
return &frame
}
func (d *Database) GetOrCreateFrame(name string) *Frame {
d.mutex.Lock()
defer d.mutex.Unlock()
frame, err := d.getFrame(name)
if err == nil {
return frame
}
return d.addFrame(name)
}
///////// SLICES
///////////////////////////////////////////////////////////////////////////
// A slice is the vertical combination of every fragment.
type Slice struct {
id int
}
func (self *Slice) Id() int {
return self.id
}
// Get a slice from a database
func (d *Database) getSlice(slice_id int) (*Slice, error) {
for _, slice := range d.slices {
if slice.id == slice_id {
return slice, nil
}
}
return nil, SliceDoesNotExistError
}
// Add a slice to a database
func (d *Database) addSlice(slice_id int) *Slice {
slice := Slice{id: slice_id}
d.slices = append(d.slices, &slice)
// add intersections
for _, frame := range d.frames {
d.AddFrameSliceIntersect(frame, &slice)
}
return &slice
}
func (d *Database) GetOrCreateSlice(slice_id int) *Slice {
d.mutex.Lock()
defer d.mutex.Unlock()
slice, err := d.getSlice(slice_id)
if err == nil {
return slice
}
return d.addSlice(slice_id)
}
///////// FRAME-SLICE INTERSECT
////////////////////////////////////////////////////////////////
type FrameSliceIntersect struct {
frame *Frame
slice *Slice
fragments []*Fragment
hashring *consistent.Consistent
}
func (d *Database) AddFrameSliceIntersect(frame *Frame, slice *Slice) *FrameSliceIntersect {
frameslice := FrameSliceIntersect{frame: frame, slice: slice}
d.frame_slice_intersects = append(d.frame_slice_intersects, &frameslice)
frameslice.hashring = consistent.New()
frameslice.hashring.NumberOfReplicas = 16
return &frameslice
}
func (d *Database) GetFrameSliceIntersect(frame *Frame, slice *Slice) (*FrameSliceIntersect, error) {
for _, frameslice := range d.frame_slice_intersects {
if frameslice.frame == frame && frameslice.slice == slice {
return frameslice, nil
}
}
log.Warn("Missing FrameSliceIntersect:", d.Name, frame, slice)
return nil, FrameSliceIntersectDoesNotExistError
}
func (self *FrameSliceIntersect) GetFragments() []*Fragment {
return self.fragments
}
func (d *Database) GetFragment(fragment_id pilosa.SUUID) (*Fragment, error) {
for _, fsi := range d.frame_slice_intersects {
f, err := fsi.GetFragment(fragment_id)
if err == nil {
return f, nil
}
}
return nil, FragmentDoesNotExistError
}
func (self *FrameSliceIntersect) GetFragment(fragment_id pilosa.SUUID) (*Fragment,
error) {
for _, fragment := range self.fragments {
if fragment.id == fragment_id {
return fragment, nil
}
}
return nil, FragmentDoesNotExistError
}
func (self *FrameSliceIntersect) AddFragment(fragment *Fragment) {
self.fragments = append(self.fragments, fragment)
self.hashring.Add(fragment.id.String())
}
///////// FRAGMENTS
//////////////////////////////////////////////////////////////////////////
// A fragment is a collection of bitmaps within a slice. The fragment contains a
// reference to the responsible node for that fragment. The node is in the form
// ip:port
type Fragment struct {
id pilosa.SUUID
process *Process
}
func (self *Fragment) GetId() pilosa.SUUID {
return self.id
}
func (self *Fragment) GetProcess() *Process {
return self.process
}
func (self *Fragment) GetProcessId() *pilosa.GUID {
return self.process.id
}
func (self *Fragment) GetLocation() *Location {
return &Location{self.process.id, self.id}
}
// rename this one
func (d *Database) GetFragmentForBitmap(slice *Slice, bitmap *Bitmap) (*Fragment, error) {
frame, err := d.getFrame(bitmap.FrameType)
if err != nil {
log.Warn("Missing FrameType", bitmap.FrameType, d.Name, slice)
log.Warn(err)
return nil, err
}
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn("Missing frame,slice", frame, slice)
log.Warn(err)
return nil, err
}
frag_id_s, err := fsi.hashring.Get(fmt.Sprintf("%d", bitmap.Id))
if err != nil {
log.Warn("ERROR FSI.GET:", bitmap.Id, bitmap.FrameType, d.Name, frame, slice)
log.Warn(err)
return nil, err
}
frag_id := pilosa.ParseSUUID(frag_id_s)
return fsi.GetFragment(frag_id)
}
func (d *Database) GetFragmentForFrameSlice(frame *Frame, slice *Slice) (*Fragment, error) {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn("Missing frame,slice", frame, slice)
log.Warn(err)
return nil, err
}
frag_id_s, err := fsi.hashring.Get("0")
// we don't need a specific bitmap in here because we're assuming the hashring only has a single element
if err != nil {
log.Warn("ERROR FSI.GET:", d.Name, frame, slice)
log.Warn(err)
return nil, err
}
frag_id := pilosa.ParseSUUID(frag_id_s)
return fsi.GetFragment(frag_id)
}
func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) (*Fragment, error) {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn(err)
return nil, err
}
return fsi.GetFragment(fragment_id)
}
func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) *Fragment {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn("database.addFragment", err)
return nil
}
fragment := Fragment{id: fragment_id}
fsi.AddFragment(&fragment)
return &fragment
}
func (d *Database) GetOrCreateFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) *Fragment {
d.mutex.Lock()
defer d.mutex.Unlock()
fragment, err := d.getFragment(frame, slice, fragment_id)
if err == nil {
return fragment
}
return d.addFragment(frame, slice, fragment_id)
}
func (f *Fragment) SetProcess(process *Process) {
f.process = process
}
func GetSlice(profile_id uint64) int {
return int(profile_id / SLICE_WIDTH)
}
///////////////////////////////////////////////////////////////////////////////////////////////
// Get a slice from a database
func (d *Database) GetSliceForProfile(profile_id uint64) (*Slice, error) {
return d.getSlice(GetSlice(profile_id))
}
type Bitmap struct {
Id uint64
FrameType string
Filter uint64
}

17
db/topology_test.go
View file

@ -1,17 +0,0 @@
package db_test
import (
"testing"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/util"
)
func TestCluster(t *testing.T) {
c := db.NewCluster()
d := c.GetOrCreateDatabase("main")
f := d.GetOrCreateFrame("general")
sl := d.GetOrCreateSlice(0)
d.GetOrCreateFragment(f, sl, util.Id())
}

123
dispatch/dispatch.go
View file

@ -1,123 +0,0 @@
package dispatch
import (
"time"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/executor"
"github.com/umbel/pilosa/query"
)
type Dispatch struct {
Executor interface {
NewJob(job *db.Message)
}
Hold interface {
Set(id *pilosa.GUID, value interface{}, timeout time.Duration)
}
Index interface {
ClearBit(fragID pilosa.SUUID, bitmapID uint64, pos uint64) (bool, error)
LoadBitmap(fragID pilosa.SUUID, bitmapID uint64, compressedBitmap string, filter uint64)
SetBit(fragID pilosa.SUUID, bitmapID uint64, pos uint64, category uint64) (bool, error)
TopFillBatch(args []pilosa.FillArgs) ([]pilosa.Pair, error)
}
Transport interface {
Receive() *db.Message
Send(message *db.Message, host *pilosa.GUID)
}
}
func NewDispatch() *Dispatch {
return &Dispatch{}
}
func (self *Dispatch) Init() error {
log.Warn("Starting Dispatcher")
return nil
}
func (self *Dispatch) Close() {
log.Warn("Shutting down Dispatcher")
}
// The Local Route
func (d *Dispatch) Run() {
log.Warn("Dispatch Run...")
for {
message := d.Transport.Receive()
switch data := message.Data.(type) {
case core.BatchRequest:
log.Trace("Dispatch.Run BatchRequest")
response := db.Message{Data: core.BatchResponse{Id: data.Id}}
d.Index.LoadBitmap(data.Fragment_id, data.Bitmap_id, data.Compressed_bitmap, data.Filter)
d.Transport.Send(&response, data.Source)
case core.BitsRequest:
log.Trace("Dispatch.Run BitsRequest")
var results []core.SBResult
result := false
for _, v := range data.Bits {
if v.SetUnset {
result, _ = d.Index.SetBit(v.Fragment_id, v.Bitmap_id, v.Profile_id, uint64(v.Filter))
} else {
result, _ = d.Index.ClearBit(v.Fragment_id, v.Bitmap_id, v.Profile_id)
}
bundle := core.SBResult{Bitmap_id: v.Bitmap_id, Frame: v.Frame, Filter: v.Filter, Profile_id: v.Profile_id, Result: result}
results = append(results, bundle)
}
response := db.Message{Data: core.BitsResponse{Id: &data.QueryId, Items: results}}
d.Transport.Send(&response, &data.ReturnProcessId)
case core.PingRequest:
log.Trace("Dispatch.Run Ping")
pong := db.Message{Data: core.PongRequest{Id: data.Id}}
d.Transport.Send(&pong, data.Source)
case db.HoldResult:
log.Trace("Dispatch.Run HoldResult")
d.Hold.Set(data.ResultId(), data.ResultData(), 30*time.Second)
case query.PortableQueryStep:
log.Trace("Dispatch.Run PortableQueryStep")
go d.Executor.NewJob(message)
case executor.TopFill:
log.Trace("Dispatch.Run TopFill")
go d.topFillHandler(message)
case core.BitsResponse:
d.Hold.Set(data.ResultId(), data.ResultData(), 30*time.Second)
default:
spew.Dump(data)
log.Warn("Unprocessed message", data)
}
}
}
func (d *Dispatch) topFillHandler(m *db.Message) {
topfill := m.Data.(executor.TopFill)
topn, err := d.Index.TopFillBatch(topfill.Args)
if err != nil {
log.Warn("TopFillHandler:", err)
}
d.Transport.Send(&db.Message{
Data: query.FillResult{
BaseQueryResult: &query.BaseQueryResult{
Id: &topfill.QueryId,
Data: topn,
},
},
}, &topfill.ReturnProcessId)
}

17
etc/default.yaml
View file

@ -1,17 +0,0 @@
id: 0d92507d-1646-44f8-be81-79d79c1df623
host: localhost
port_tcp: 12001
port_http: 15001
log_path: /tmp
fragment_base: /tmp/single
storage_backend: cassandra
cassandra_host: localhost
cassandra_keyspace: pilosa
supported_frames:
- b.n
- t.t
- l.n
- d
- p.n
etcd_servers:
- http://127.0.0.1:4001

316
executor.go Normal file
View file

@ -0,0 +1,316 @@
package pilosa
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa/internal"
"github.com/umbel/pilosa/pql"
)
// DefaultFrame is the frame used if one is not specified.
const DefaultFrame = "general"
// ErrDatabaseRequired is returned when no database is specified.
var ErrDatabaseRequired = errors.New("database required")
// Executor recursively executes calls in a PQL query across all slices.
type Executor struct {
index *Index
// Local hostname.
Host string
// Cluster configuration
Cluster *Cluster
// Client used for remote HTTP requests.
HTTPClient *http.Client
}
// NewExecutor returns a new instance of Executor.
func NewExecutor(index *Index) *Executor {
return &Executor{
index: index,
HTTPClient: http.DefaultClient,
}
}
// Index returns the index that the executor runs against.
func (e *Executor) Index() *Index { return e.index }
// Execute executes a PQL query.
func (e *Executor) Execute(db string, q *pql.Query, slices []uint64) (interface{}, error) {
// Verify that a database is set.
if db == "" {
return nil, ErrDatabaseRequired
}
// If slices aren't specified, then include all of them.
if len(slices) == 0 {
// Round up the number of slices.
sliceN := (e.index.SliceN() % uint64(len(e.Cluster.Nodes))) + uint64(len(e.Cluster.Nodes))
// Generate a slices of all slices.
slices = make([]uint64, sliceN+1)
for i := range slices {
slices[i] = uint64(i)
}
}
return e.executeCall(db, q.Root, slices)
}
// executeCall executes a call.
func (e *Executor) executeCall(db string, c pql.Call, slices []uint64) (interface{}, error) {
switch c := c.(type) {
case pql.BitmapCall:
return e.executeBitmapCall(db, c, slices)
case *pql.Count:
return e.executeCount(db, c, slices)
case *pql.TopN:
return e.executeTopN(db, c, slices)
default:
panic("unreachable")
}
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint64) (*Bitmap, error) {
other := NewBitmap()
for node, nodeSlices := range e.slicesByNode(slices) {
// Execute locally if the hostname matches.
if node.Host == e.Host {
for _, slice := range nodeSlices {
bm, err := e.executeBitmapCallSlice(db, c, slice)
if err != nil {
return nil, err
}
other.Merge(bm)
}
continue
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices)
if err != nil {
return nil, err
}
other.Merge(res.(*Bitmap))
}
return other, nil
}
// executeBitmapCallSlice executes a bitmap call for a single slice.
func (e *Executor) executeBitmapCallSlice(db string, c pql.BitmapCall, slice uint64) (*Bitmap, error) {
switch c := c.(type) {
case *pql.Difference:
return e.executeDifferenceSlice(db, c, slice)
case *pql.Get:
return e.executeGetSlice(db, c, slice)
case *pql.Intersect:
return e.executeIntersectSlice(db, c, slice)
case *pql.Range:
return e.executeRangeSlice(db, c, slice)
case *pql.Union:
return e.executeUnionSlice(db, c, slice)
default:
panic("unreachable")
}
}
// executeTopN executes a top-n() call.
func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, error) {
panic("FIXME: calculate top n from each slice")
}
// executeDifferenceSlice executes a difference() call for a local slice.
func (e *Executor) executeDifferenceSlice(db string, c *pql.Difference, slice uint64) (*Bitmap, error) {
var other *Bitmap
for i, input := range c.Inputs {
bm, err := e.executeBitmapCallSlice(db, input, slice)
if err != nil {
return nil, err
}
if i == 0 {
other = bm
} else {
other = other.Difference(bm)
}
}
return other, nil
}
func (e *Executor) executeGetSlice(db string, c *pql.Get, slice uint64) (*Bitmap, error) {
frame := c.Frame
if frame == "" {
frame = DefaultFrame
}
f := e.Index().Fragment(db, frame, slice)
return f.Bitmap(c.ID), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
func (e *Executor) executeIntersectSlice(db string, c *pql.Intersect, slice uint64) (*Bitmap, error) {
var other *Bitmap
for i, input := range c.Inputs {
bm, err := e.executeBitmapCallSlice(db, input, slice)
if err != nil {
return nil, err
}
if i == 0 {
other = bm
} else {
other = other.Intersect(bm)
}
}
return other, nil
}
// executeRangeSlice executes a range() call for a local slice.
func (e *Executor) executeRangeSlice(db string, c *pql.Range, slice uint64) (*Bitmap, error) {
panic("FIXME")
}
// executeUnionSlice executes a union() call for a local slice.
func (e *Executor) executeUnionSlice(db string, c *pql.Union, slice uint64) (*Bitmap, error) {
var other *Bitmap
for i, input := range c.Inputs {
bm, err := e.executeBitmapCallSlice(db, input, slice)
if err != nil {
return nil, err
}
if i == 0 {
other = bm
} else {
other = other.Union(bm)
}
}
return other, nil
}
// executeCount executes a count() call.
func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint64, error) {
var n uint64
for node, nodeSlices := range e.slicesByNode(slices) {
// Execute locally if the hostname matches.
if node.Host == e.Host {
for _, slice := range nodeSlices {
bm, err := e.executeBitmapCallSlice(db, c.Input, slice)
if err != nil {
return 0, err
}
n += bm.Count()
}
continue
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices)
if err != nil {
return 0, err
}
n += res.(uint64)
}
return n, nil
}
// exec executes a PQL query remotely for a set of slices on a node.
func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64) (result interface{}, err error) {
// Encode request object.
buf, err := proto.Marshal(&internal.QueryRequest{
DB: proto.String(db),
Query: proto.String(q.String()),
Slices: slices,
})
if err != nil {
return nil, err
}
// Create HTTP request.
req, err := http.NewRequest("POST", (&url.URL{
Scheme: "http",
Host: node.Host,
Path: "/query",
}).String(), bytes.NewReader(buf))
if err != nil {
return nil, err
}
// Require protobuf encoding.
req.Header.Set("Accept", "application/x-protobuf")
req.Header.Set("Content-Type", "application/x-protobuf")
// Send request to remote node.
resp, err := e.HTTPClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// Read response into buffer.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
// Check status code.
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body)
}
// Decode response object.
var pb internal.QueryResponse
if err := proto.Unmarshal(body, &pb); err != nil {
return nil, err
}
// Return an error, if specified on response.
if err := decodeError(pb.GetErr()); err != nil {
return nil, err
}
// Return appropriate data for the query.
switch q.Root.(type) {
case pql.BitmapCall:
return decodeBitmap(pb.GetBitmap()), nil
case *pql.TopN:
return decodePairs(pb.GetPairs()), nil
case *pql.Count:
return pb.GetN(), nil
default:
panic(fmt.Sprintf("invalid node for remote exec: %T", q.Root))
}
}
// slicesByNode returns a mapping of nodes to slices.
//
// NOTE: Currently the only primary node is used.
func (e *Executor) slicesByNode(slices []uint64) map[*Node][]uint64 {
m := make(map[*Node][]uint64)
for _, slice := range slices {
nodes := e.Cluster.SliceNodes(slice)
node := nodes[0]
m[node] = append(m[node], slice)
}
return m
}
// decodeError returns an error representation of s if s is non-blank.
// Returns nil if s is blank.
func decodeError(s string) error {
if s == "" {
return nil
}
return errors.New(s)
}

800
executor/executor.go
View file

@ -1,800 +0,0 @@
package executor
import (
"encoding/gob"
"sort"
"time"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/query"
)
const DefaultTimeout = 30 * time.Second
type Executor struct {
inbox chan *db.Message
ID pilosa.GUID
Cluster *db.Cluster
ProcessMap *core.ProcessMap
PluginsPath string
Hold interface {
Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error)
Set(id *pilosa.GUID, value interface{}, timeout time.Duration)
}
Index interface {
ClearBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64) (bool, error)
Count(frag_id pilosa.SUUID, bitmap pilosa.BitmapHandle) (uint64, error)
Difference(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
FromBytes(frag_id pilosa.SUUID, bytes []byte) (pilosa.BitmapHandle, error)
Get(frag_id pilosa.SUUID, bitmap_id uint64) (pilosa.BitmapHandle, error)
GetBytes(frag_id pilosa.SUUID, bh pilosa.BitmapHandle) ([]byte, error)
Intersect(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
Range(frag_id pilosa.SUUID, bitmap_id uint64, start, end time.Time) (pilosa.BitmapHandle, error)
SetBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error)
TopN(frag_id pilosa.SUUID, bh pilosa.BitmapHandle, n int, categories []uint64) ([]pilosa.Pair, error)
TopNAll(frag_id pilosa.SUUID, n int, categories []uint64) ([]pilosa.Pair, error)
Union(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
}
TopologyMapper interface {
MakeFragments(db string, slice_int int) error
}
Transport interface {
Send(*db.Message, *pilosa.GUID)
}
}
func NewExecutor(id pilosa.GUID) *Executor {
log.Trace("NewExector")
return &Executor{inbox: make(chan *db.Message)}
}
func (self *Executor) Init() error {
log.Trace("Executor.Init()")
return nil
}
func (self *Executor) Close() {
log.Trace("Executor.Close()")
}
func (self *Executor) NewJob(job *db.Message) {
log.Trace("NewJob", job)
switch job.Data.(type) {
case query.CountQueryStep:
self.CountQueryStepHandler(job)
case query.TopNQueryStep:
self.TopNQueryStepHandler(job)
case query.UnionQueryStep:
self.UnionQueryStepHandler(job)
case query.IntersectQueryStep:
self.IntersectQueryStepHandler(job)
case query.DifferenceQueryStep:
self.DifferenceQueryStepHandler(job)
case query.CatQueryStep:
self.CatQueryStepHandler(job)
case query.GetQueryStep:
self.GetQueryStepHandler(job)
case query.SetQueryStep:
self.SetQueryStepHandler(job)
case query.ClearQueryStep:
self.ClearQueryStepHandler(job)
case query.RangeQueryStep:
self.RangeQueryStepHandler(job)
case query.StashQueryStep:
self.StashQueryStepHandler(job)
default:
log.Warn("unknown")
log.Warn(spew.Sdump(job.Data))
}
}
func (self *Executor) CountQueryStepHandler(msg *db.Message) {
log.Trace("CountQueryStepHandler")
//spew.Dump("COUNT QUERYSTEP")
qs := msg.Data.(query.CountQueryStep)
input := qs.Input
value, _ := self.Hold.Get(input, DefaultTimeout)
var bh pilosa.BitmapHandle
switch val := value.(type) {
case pilosa.BitmapHandle:
bh = val
case []byte:
bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val)
}
count, err := self.Index.Count(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
//spew.Dump("SLICE COUNT", count)
result_message := db.Message{
Data: query.CountQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: count},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) TopNQueryStepHandler(msg *db.Message) {
qs := msg.Data.(query.TopNQueryStep)
var bh pilosa.BitmapHandle
var topnPackage TopNPackage
// if we have an input, hold for it. if we don't, we assume an all() query
if qs.Input == nil {
topn, err := self.Index.TopNAll(qs.Location.FragmentId, qs.N*2, qs.Filters)
if err != nil {
log.Warn(spew.Sdump(err))
}
topnPackage = TopNPackage{*qs.Location.ProcessId, qs.Location.FragmentId, topn, bh}
} else {
input := qs.Input
value, _ := self.Hold.Get(input, 10)
//var bh pilosa.BitmapHandle
switch val := value.(type) {
case pilosa.BitmapHandle:
bh = val
case []byte:
bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val)
}
topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N*2, qs.Filters)
if err != nil {
log.Warn(spew.Sdump(err))
}
topnPackage = TopNPackage{*qs.Location.ProcessId, qs.Location.FragmentId, topn, bh}
}
result_message := db.Message{
Data: query.TopNQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: topnPackage},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) UnionQueryStepHandler(msg *db.Message) {
log.Trace("UnionQueryStepHandler")
//spew.Dump("UNION QUERYSTEP")
qs := msg.Data.(query.UnionQueryStep)
var handles []pilosa.BitmapHandle
// create a list of bitmap handles
for _, input := range qs.Inputs {
value, _ := self.Hold.Get(input, DefaultTimeout)
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
handles = append(handles, bh)
}
}
bh, err := self.Index.Union(qs.Location.FragmentId, handles)
if err != nil {
spew.Dump(err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
result = bm
}
result_message := db.Message{
Data: query.UnionQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) IntersectQueryStepHandler(msg *db.Message) {
log.Trace("IntersectQueryStepHandler")
//spew.Dump("INTERSECT QUERYSTEP")
qs := msg.Data.(query.IntersectQueryStep)
var handles []pilosa.BitmapHandle
// create a list of bitmap handles
for _, input := range qs.Inputs {
value, _ := self.Hold.Get(input, DefaultTimeout)
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
handles = append(handles, bh)
}
}
bh, err := self.Index.Intersect(qs.Location.FragmentId, handles)
if err != nil {
spew.Dump(err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
result = bm
}
result_message := db.Message{
Data: query.IntersectQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) DifferenceQueryStepHandler(msg *db.Message) {
log.Trace("DifferenceQueryStepHandler")
//spew.Dump("DIFFERENCE QUERYSTEP")
qs := msg.Data.(query.DifferenceQueryStep)
var handles []pilosa.BitmapHandle
// create a list of bitmap handles
for _, input := range qs.Inputs {
value, _ := self.Hold.Get(input, DefaultTimeout)
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
handles = append(handles, bh)
}
}
bh, err := self.Index.Difference(qs.Location.FragmentId, handles)
if err != nil {
spew.Dump(err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
result = bm
}
result_message := db.Message{
Data: query.DifferenceQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) CatQueryStepHandler(msg *db.Message) {
log.Trace("CatQueryStepHandler")
qs := msg.Data.(query.CatQueryStep)
var handles []pilosa.BitmapHandle
return_type := "bitmap-handles"
var sum uint64
merge_map := make(map[uint64]uint64)
slice_map := make(map[uint64]map[pilosa.SUUID]struct{})
all_slice := make(map[pilosa.SUUID]struct {
process pilosa.GUID
handle pilosa.BitmapHandle
})
// either create a list of bitmap handles to cat (i.e. union), or sum the integer values
part := make(chan interface{})
num_parts := len(qs.Inputs)
for _, input := range qs.Inputs {
go func(id *pilosa.GUID, part chan interface{}) {
value, _ := self.Hold.Get(id, DefaultTimeout)
part <- value
}(input, part)
}
//for _, input := range qs.Inputs {
check_pair := false
for i := 0; i < num_parts; i++ {
value := <-part
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
handles = append(handles, bh)
case uint64:
//spew.Dump(val)
return_type = "sum"
sum += val
case TopNPackage:
return_type = "pair-list"
var e struct{}
for _, pair := range val.Pairs {
//merge_map[pair.Key] += pair.Count
if pair.Key == 0 {
continue //skip
}
merge_map[pair.Key] += pair.Count
mm, ok := slice_map[pair.Key]
if !ok {
mm = make(map[pilosa.SUUID]struct{})
slice_map[pair.Key] = mm
}
mm[val.FragmentId] = e
}
all_slice[val.FragmentId] = struct {
process pilosa.GUID
handle pilosa.BitmapHandle
}{val.ProcessId, val.HBitmap}
check_pair = true
}
}
if check_pair { //no point in doing this for non top-n handling
tasks := BuildTask(merge_map, slice_map, all_slice)
self.FetchMissing(tasks)
for k, v := range self.GatherResults(tasks) {
merge_map[k] += v
}
}
// either return the sum, or return the compressed bitmap resulting from the cat (union)
var result interface{}
if return_type == "sum" {
result = sum
} else if return_type == "bitmap-handles" {
bh, err := self.Index.Union(qs.Location.FragmentId, handles)
result, err = self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
} else if return_type == "pair-list" {
rank_list := make(pilosa.RankList, 0, len(merge_map))
for k, v := range merge_map {
if k == 0 || v == 0 {
continue //shouldn't be getting 0 keys or values anyway
}
rank := new(pilosa.Rank)
rank.Pair = &pilosa.Pair{Key: k, Count: v}
rank_list = append(rank_list, rank)
}
sort.Sort(rank_list) // kinda seems like this copy is wasteful..i'll ponder
items_size := min(len(merge_map), qs.N)
pair_list := make([]pilosa.Pair, 0, items_size+1)
for i, r := range rank_list {
if i < items_size {
pair_list = append(pair_list, *r.Pair)
} else {
break
}
}
result = pair_list
} else {
result = "NONE"
}
result_message := db.Message{
Data: query.CatQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) SendRequest(process_id pilosa.GUID, t *Task) {
args := make([]pilosa.FillArgs, len(t.f), len(t.f))
for _, v := range t.f {
args = append(args, v)
}
msg := new(db.Message)
p, _ := self.ProcessMap.GetProcess(&self.ID)
msg.Data = TopFill{args, p.Id(), t.hold_id, process_id}
self.Transport.Send(msg, &process_id)
}
func (self *Executor) FetchMissing(tasks map[pilosa.GUID]*Task) {
for k, v := range tasks {
go self.SendRequest(k, v)
}
}
func (self *Executor) GatherResults(tasks map[pilosa.GUID]*Task) map[uint64]uint64 {
results := make(map[uint64]uint64)
answers := make(chan []pilosa.Pair)
for _, task := range tasks {
go func(id pilosa.GUID) {
value, err := self.Hold.Get(&id, 10) //eiher need to be the frame process or the handler process?
if value == nil {
log.Warn("Bad TopN Result:", err)
empty := make([]pilosa.Pair, 0, 0)
answers <- empty
} else {
answers <- value.([]pilosa.Pair)
}
}(task.hold_id)
}
for i := 0; i < len(tasks); i++ {
batch := <-answers
for _, pair := range batch {
results[pair.Key] += pair.Count
}
}
close(answers)
return results
}
func (self *Executor) GetQueryStepHandler(msg *db.Message) {
qs := msg.Data.(query.GetQueryStep)
//spew.Dump("GET QUERYSTEP")
bh, err := self.Index.Get(qs.Location.FragmentId, qs.Bitmap.Id)
if err != nil {
spew.Dump(err)
log.Error("GetQueryStepHandler1", qs.Location.FragmentId.String(), qs.Bitmap.Id)
log.Error("GetQueryStepHandler2", err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
log.Error("GetQueryStepHandlerr3", qs.Location.FragmentId.String(), qs.Bitmap.Id)
log.Error("GetQueryStepHandler4", err)
}
result = bm
}
result_message := db.Message{
Data: query.GetQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) SetQueryStepHandler(msg *db.Message) {
//spew.Dump("SET QUERYSTEP")
qs := msg.Data.(query.SetQueryStep)
result, _ := self.Index.SetBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId, qs.Bitmap.Filter)
result_message := db.Message{
Data: query.SetQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) ClearQueryStepHandler(msg *db.Message) {
//spew.Dump("SET QUERYSTEP")
qs := msg.Data.(query.ClearQueryStep)
result, _ := self.Index.ClearBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId)
result_message := db.Message{
Data: query.ClearQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) RangeQueryStepHandler(msg *db.Message) {
qs := msg.Data.(query.RangeQueryStep)
//spew.Dump("RANDE QUERYSTEP")
bh, err := self.Index.Range(qs.Location.FragmentId, qs.Bitmap.Id, qs.Start, qs.End)
if err != nil {
spew.Dump(err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
result = bm
}
result_message := db.Message{
Data: query.RangeQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) StashQueryStepHandler(msg *db.Message) {
log.Trace("StashQueryStepHandler")
qs := msg.Data.(query.StashQueryStep)
part := make(chan interface{})
num_parts := len(qs.Inputs)
for _, input := range qs.Inputs {
go func(id *pilosa.GUID, part chan interface{}) {
value, _ := self.Hold.Get(id, DefaultTimeout)
part <- value
}(input, part)
}
//just collect all the handles and return them
result := query.NewStash() //query.Stash{make([]query.CacheItem, 0), false}
for i := 0; i < num_parts; i++ {
value := <-part
switch val := value.(type) {
case pilosa.BitmapHandle:
log.Info("STASH ADDING HANDLE", val)
//not sure what to do here....
//result.Handles = append(result.Handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
item := query.CacheItem{FragmentId: qs.Location.FragmentId, Handle: bh}
result.Stash = append(result.Stash, item)
case query.Stash:
result.Stash = append(result.Stash, val.Stash...)
default:
log.Warn("UNEXCPECTED MESSAGE", value)
}
}
result_message := db.Message{
Data: query.StashQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) RunQueryTest(database_name string, pql string) string {
return pql
}
func (self *Executor) runQuery(database *db.Database, qry *query.Query) error {
log.Trace("Executor.runQuery", database, qry)
process, err := self.ProcessMap.GetProcess(&self.ID)
if err != nil {
return err
}
process_id := process.Id()
fragment_id := pilosa.SUUID(0)
destination := db.Location{ProcessId: &process_id, FragmentId: fragment_id}
query_plan, err := query.QueryPlanForQuery(database, qry, &destination)
if err != nil {
switch obj := err.(type) {
case *query.FragmentNotFound:
self.TopologyMapper.MakeFragments(obj.Db, obj.Slice)
}
self.Hold.Set(qry.Id, err, 30)
return err
}
// loop over the query steps and send to Transport
for _, qs := range *query_plan {
msg := new(db.Message)
msg.Data = qs
switch step := qs.(type) {
case query.PortableQueryStep:
loc := step.GetLocation()
if loc != nil {
self.Transport.Send(msg, loc.ProcessId)
} else {
log.Warn("Problem with querystep(nil location)", spew.Sdump(step))
}
}
}
return nil
}
func (self *Executor) RunPQL(database_name string, pql string) (interface{}, error) {
log.Trace("Executor.RunPQL", database_name, pql)
database := self.Cluster.GetOrCreateDatabase(database_name)
// see if the outer query function is a custom query
reserved_functions := stringSlice{"get", "set", "clear", "union", "intersect", "difference", "count", "top-n", "mask", "range", "stash", "recall"}
tokens, err := query.Lex(pql)
if err != nil {
return nil, err
}
outer_token := tokens[0].Text
if reserved_functions.pos(outer_token) != -1 {
qry, err := query.QueryForTokens(tokens)
if err != nil {
return nil, err
}
go self.runQuery(database, qry)
var final interface{}
final, err = self.Hold.Get(qry.Id, 10)
if err != nil {
return nil, err
}
return final, nil
} else { //want to refactor this down to just RunPlugin(tokens)
plugins_file := self.PluginsPath + "/" + outer_token + ".js"
filter, filters := query.TokensToFilterStrings(tokens)
query_list := GetPlugin(plugins_file, filter, filters).(query.PqlList)
for i, _ := range query_list {
qry, err := query.QueryForPQL(query_list[i].PQL)
if err != nil {
return nil, err
}
go self.runQuery(database, qry)
query_list[i].Id = qry.Id
}
final_result := make(map[string]interface{})
result := make(chan struct {
final interface{}
label string
err error
})
x := 0
for i, _ := range query_list {
x++
go func(q query.PqlListItem, reply chan struct {
final interface{}
label string
err error
}) {
final, err := self.Hold.Get(q.Id, 10)
result <- struct {
final interface{}
label string
err error
}{final, q.Label, err}
}(query_list[i], result)
if err != nil {
out := spew.Sdump(err)
log.Warn(out)
}
}
for z := 0; z < x; z++ {
ans := <-result
final_result[ans.label] = ans.final
}
return final_result, nil
}
}
func init() {
gob.Register(TopNPackage{})
gob.Register(TopFill{})
}
type TopNPackage struct {
ProcessId pilosa.GUID
FragmentId pilosa.SUUID
Pairs []pilosa.Pair
HBitmap pilosa.BitmapHandle
}
type TopFill struct {
Args []pilosa.FillArgs
ReturnProcessId pilosa.GUID
QueryId pilosa.GUID
DestProcessId pilosa.GUID
}
type Task struct {
processid pilosa.GUID
f map[pilosa.SUUID]pilosa.FillArgs
hold_id pilosa.GUID
}
func newtask(p pilosa.GUID) *Task {
result := new(Task)
result.processid = p
result.f = make(map[pilosa.SUUID]pilosa.FillArgs)
result.hold_id = pilosa.NewGUID()
return result
}
func (t *Task) Add(frag pilosa.SUUID, bitmap_id uint64, handle pilosa.BitmapHandle) {
fa, ok := t.f[frag]
if !ok {
fa = pilosa.FillArgs{Frag_id: frag, Handle: handle, Bitmaps: make([]uint64, 0, 0)}
}
fa.Bitmaps = append(fa.Bitmaps, bitmap_id)
t.f[frag] = fa
}
func BuildTask(merge_map map[uint64]uint64,
slice_map map[uint64]map[pilosa.SUUID]struct{},
total_fragments map[pilosa.SUUID]struct {
process pilosa.GUID
handle pilosa.BitmapHandle
}) map[pilosa.GUID]*Task {
tasks := make(map[pilosa.GUID]*Task)
for bitmap_id, _ := range merge_map { //for all brands
//for fragment_id, reported_fragments := range slice_map[bitmap_id] { //find missing fragments
reporting_fragments := slice_map[bitmap_id]
//id slice ==> SUUID,BitmapHandle
for _, p := range missing(reporting_fragments, total_fragments) {
task, ok := tasks[p.process]
if !ok {
task = newtask(p.process)
tasks[p.process] = task
}
task.Add(p.fragment, bitmap_id, p.handle)
}
//}
}
return tasks
}
type hole struct {
process pilosa.GUID
handle pilosa.BitmapHandle
fragment pilosa.SUUID
}
func missing(fids map[pilosa.SUUID]struct{}, all map[pilosa.SUUID]struct {
process pilosa.GUID
handle pilosa.BitmapHandle
}) []hole {
results := make([]hole, 0, 0)
for k, v := range all {
_, ok := fids[k]
if !ok {
results = append(results, hole{v.process, v.handle, k})
}
}
return results
}
func (self *TopFill) GetId() *pilosa.GUID {
return &self.QueryId
}
func (self *TopFill) GetLocation() *db.Location {
return &db.Location{ProcessId: &self.DestProcessId, FragmentId: 0} //this message is a broadcast to many fragments so i'm choosing fragmentzero
}
func (self *Executor) Run() {
log.Warn("Executor Run...")
}
type stringSlice []string
func (slice stringSlice) pos(value string) int {
for p, v := range slice {
if v == value {
return p
}
}
return -1
}
func min(a, b int) int {
if a < b {
return a
}
return b
}

57
executor/utils.go
View file

@ -1,57 +0,0 @@
package executor
import (
"bytes"
"io/ioutil"
"strings"
"github.com/davecgh/go-spew/spew"
"github.com/robertkrimen/otto"
"github.com/umbel/pilosa/query"
)
func GetPlugin(file_name string, filter string, filters []string) interface{} {
file_data, err := ioutil.ReadFile(file_name)
if err != nil {
spew.Dump(err)
}
s := string(file_data[:])
// convert the list of filters in to a string array
var buffer bytes.Buffer
if len(filters) > 0 {
buffer.WriteString("['")
buffer.WriteString(strings.Join(filters, "','"))
buffer.WriteString("']")
} else {
buffer.WriteString("[]")
}
js := "query_list = (" + s + ")('" + filter + "', " + buffer.String() + ");"
Otto := otto.New()
Otto.Run(js)
query_objects, err := Otto.Get("query_list")
query_list_interface, err := query_objects.Export()
if err != nil {
spew.Dump(err)
}
var query_list query.PqlList
// ql is []interface{}
switch ql := query_list_interface.(type) {
case []interface{}:
// q is map[string]interface{}
for i, _ := range ql {
q := ql[i].(map[string]interface{})
query_list = append(query_list, query.PqlListItem{Label: q["label"].(string), PQL: q["pql"].(string)})
}
default:
spew.Dump("DEFAULT")
}
return query_list
}

198
executor_test.go Normal file
View file

@ -0,0 +1,198 @@
package pilosa_test
import (
"reflect"
"strings"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/pql"
)
// Ensure a get query can be executed.
func TestExecutor_Execute_Get(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "f", 0).Bitmap(10).SetBit(3)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit(SliceWidth + 1)
if res, err := e.Execute("d", MustParse(`get(id=10, frame=f)`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 8 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 2 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
}
}
// Ensure a difference query can be executed.
func TestExecutor_Execute_Difference(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(1)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(2)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(3)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(2)
if res, err := e.Execute("d", MustParse(`difference(get(id=10), get(id=11))`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 1 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 10 { // b1010
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
}
}
// Ensure an intersect query can be executed.
func TestExecutor_Execute_Intersect(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(1)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(SliceWidth + 1)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(SliceWidth + 2)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(1)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(2)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(SliceWidth + 2)
if res, err := e.Execute("d", MustParse(`intersect(get(id=10), get(id=11))`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 2 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 4 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
}
}
// Ensure a union query can be executed.
func TestExecutor_Execute_Union(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(0)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(SliceWidth + 1)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(SliceWidth + 2)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(2)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(SliceWidth + 2)
if res, err := e.Execute("d", MustParse(`union(get(id=10), get(id=11))`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 5 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 6 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
}
}
// Ensure a count query can be executed.
func TestExecutor_Execute_Count(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "f", 0).Bitmap(10).SetBit(3)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit(SliceWidth + 1)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit(SliceWidth + 2)
if n, err := e.Execute("d", MustParse(`count(get(id=10, frame=f))`), nil); err != nil {
t.Fatal(err)
} else if n != uint64(3) {
t.Fatalf("unexpected n: %d", n)
}
}
// Ensure a remote query can return a bitmap.
func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c := NewCluster(2)
// Create secondary server and update second cluster node.
s := NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `get(id=10, frame=f)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 2}) {
t.Fatalf("unexpected slices: %+v", slices)
}
// Set bits in slice 0 & 2.
bm := pilosa.NewBitmap()
bm.SetBit((0 * SliceWidth) + 1)
bm.SetBit((0 * SliceWidth) + 2)
bm.SetBit((2 * SliceWidth) + 4)
return bm, nil
}
// Create local executor data.
// The local node owns slice 1.
e := NewExecutor(c)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit((1 * SliceWidth) + 1)
if res, err := e.Execute("d", MustParse(`get(id=10, frame=f)`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 3 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 6 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 2 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
}
}
// Ensure a remote query can return a count.
func TestExecutor_Execute_Remote_Count(t *testing.T) {
c := NewCluster(2)
// Create secondary server and update second cluster node.
s := NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return uint64(10), nil
}
// Create local executor data. The local node owns slice 1.
e := NewExecutor(c)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit((1 * SliceWidth) + 1)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit((1 * SliceWidth) + 2)
if n, err := e.Execute("d", MustParse(`count(get(id=10, frame=f))`), nil); err != nil {
t.Fatal(err)
} else if n != uint64(12) {
t.Fatalf("unexpected n: %d", n)
}
}
// Executor represents a test wrapper for pilosa.Executor.
type Executor struct {
*pilosa.Executor
}
// NewExecutor returns a new instance of Executor.
// The executor always matches the hostname of the first cluster node.
func NewExecutor(cluster *pilosa.Cluster) *Executor {
e := &Executor{
Executor: pilosa.NewExecutor(pilosa.NewIndex()),
}
e.Cluster = cluster
e.Host = cluster.Nodes[0].Host
return e
}
// MustParse parses s into a PQL query. Panic on error.
func MustParse(s string) *pql.Query {
q, err := pql.NewParser(strings.NewReader(s)).Parse()
if err != nil {
panic(err)
}
return q
}

329
fragment.go
View file

@ -1,213 +1,230 @@
package pilosa
import (
"sort"
"strings"
"sync"
"time"
log "github.com/cihub/seelog"
"github.com/golang/groupcache/lru"
)
// SliceWidth is the number of profile IDs in a slice.
const SliceWidth = 65536
// Fragment represents the intersection of a frame and slice in a database.
type Fragment struct {
mu sync.Mutex
id SUUID
slice int
impl Pilosa
cache *lru.Cache
mu sync.Mutex
// Provide an autoincrementing index for bitmap handles.
seq uint64
// Composite identifiers
db string
frame string
slice uint64
// Stats for how many messages have been processed.
stats FragmentStats
// Bitmap cache.
cache Cache
}
func NewFragment(id SUUID, db string, slice int, frame string) *Fragment {
storage := NewStorage(Backend, StorageOptions{
DB: db,
Slice: slice,
Frame: frame,
FragmentID: id,
LevelDBPath: LevelDBPath,
})
var impl Pilosa
if strings.HasSuffix(frame, ".n") {
impl = NewBrand(db, frame, slice, storage, 50000, 45000, 100)
} else {
impl = NewGeneral(db, frame, slice, storage)
}
return &Fragment{
id: id,
cache: lru.New(50000),
impl: impl,
// NewFragment returns a new instance of Fragment.
func NewFragment(db, frame string, slice uint64) *Fragment {
f := &Fragment{
db: db,
frame: frame,
slice: slice,
}
}
func (f *Fragment) Bitmap(bh BitmapHandle) (*Bitmap, bool) {
f.mu.Lock()
defer f.mu.Unlock()
return f.bitmap(bh)
}
func (f *Fragment) bitmap(bh BitmapHandle) (*Bitmap, bool) {
bm, ok := f.cache.Get(bh)
if ok && bm != nil {
return bm.(*Bitmap), ok
// Determine cache type from frame name.
if strings.HasSuffix(frame, ".n") {
c := NewRankCache()
c.ThresholdLength = 50000
c.ThresholdIndex = 45000
f.cache = c
} else {
f.cache = NewLRUCache(50000)
}
return NewBitmap(), false //cache fail
return f
}
func (f *Fragment) exists(bitmapID uint64) bool {
// Bitmap returns a bitmap by ID.
func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap {
f.mu.Lock()
defer f.mu.Unlock()
return f.impl.Exists(bitmapID)
return f.bitmap(bitmapID)
}
func (f *Fragment) bitmap(bitmapID uint64) *Bitmap {
// Read from cache.
if bm, ok := f.cache.Get(bitmapID); ok {
return bm
}
// Read from storage engine.
// bm, filter := f.storage.Fetch(bitmapID, f.db, f.frame, f.slice)
bm := NewBitmap()
f.cache.Add(bitmapID, 0 /*filter*/, bm)
return bm
}
func (f *Fragment) TopNAll(n int, categories []uint64) []Pair {
f.mu.Lock()
defer f.mu.Unlock()
return f.impl.TopNAll(n, categories)
}
f.cache.Invalidate()
func (f *Fragment) TopN(bitmap BitmapHandle, n int, categories []uint64) []Pair {
f.mu.Lock()
defer f.mu.Unlock()
bm, ok := f.cache.Get(bitmap)
if ok {
return f.impl.TopN(bm.(*Bitmap), n, categories)
// Create a set of categories.
m := make(map[uint64]struct{})
for _, v := range categories {
m[v] = struct{}{}
}
return nil
}
func (f *Fragment) NewHandle(bitmapID uint64) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
return f.allocHandle(f.impl.Get(bitmapID))
}
// Iterate over rankings and add to results until we have enough.
var results []Pair
for _, pair := range f.cache.Pairs() {
// Skip if categories are specified but category is not found.
if _, ok := m[pair.category]; (len(categories) > 0 && !ok) || pair.Count <= 0 {
continue
}
func (f *Fragment) AllocHandle(bm *Bitmap) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
return f.allocHandle(bm)
}
// Append pair.
results = append(results, pair)
func (f *Fragment) allocHandle(bm *Bitmap) BitmapHandle {
handle := f.nextHandle()
f.cache.Add(handle, bm)
return handle
}
func (f *Fragment) nextHandle() BitmapHandle {
millis := uint64(time.Now().UTC().UnixNano())
id := millis << (64 - 41)
id |= uint64(f.slice) << (64 - 41 - 13)
id |= f.seq % 1024
f.seq += 1
return BitmapHandle(id)
}
func (f *Fragment) Union(bitmaps []BitmapHandle) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
result := NewBitmap()
for i, id := range bitmaps {
bm, _ := f.bitmap(id)
if i == 0 {
result = bm
} else {
result = result.Union(bm)
// Exit when we have enough pairs.
if len(results) >= n {
break
}
}
return f.allocHandle(result)
return results
}
func (f *Fragment) build_time_range_bitmap(bitmapID uint64, start, end time.Time) BitmapHandle {
func (f *Fragment) TopN(src *Bitmap, n int, categories []uint64) []Pair {
f.mu.Lock()
defer f.mu.Unlock()
result := NewBitmap()
for i, bid := range GetRange(start, end, bitmapID) {
bm := f.impl.Get(bid)
if i == 0 {
result = bm
} else {
result = result.Union(bm)
// Resort rank, if necessary.
f.cache.Invalidate()
// Create a set of categories.
set := make(map[uint64]struct{})
for _, v := range categories {
set[v] = struct{}{}
}
var results []Pair
var x int
breakout := 1000
// Iterate over rankings.
rankings := f.cache.Pairs()
for i, pair := range rankings {
// Skip if category not found.
if len(set) > 0 {
if _, ok := set[pair.category]; !ok {
continue
}
}
// Only append if there are intersecting bits with source bitmap.
bc := src.IntersectionCount(pair.bitmap)
if bc > 0 {
results = append(results, Pair{
Key: pair.Key,
Count: bc,
category: pair.category,
})
}
x = i
// Exit when we have enough.
if len(results) > n {
break
}
}
return f.AllocHandle(result)
}
func (f *Fragment) Intersect(bitmaps []BitmapHandle) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
// Sort results by ranking.
sort.Sort(Pairs(results))
var result *Bitmap
for i, id := range bitmaps {
bm, _ := f.bitmap(id)
if i == 0 {
result = bm.Clone()
} else {
result = result.Intersection(bm)
if len(results) < n {
return results
}
end := len(results) - 1
o := results[end]
threshold := o.Count
if threshold <= 10 {
return results
}
results = append(results, o)
for i := x + 1; i < len(rankings); i++ {
o = rankings[i]
if len(set) > 0 {
if _, ok := set[o.category]; !ok {
continue
}
}
// Need something to do with the size of initial bitmap
if len(results) > breakout || o.Count < threshold {
break
}
bc := src.IntersectionCount(o.bitmap)
if bc > threshold {
if results[end-1].Count > bc {
results[end] = Pair{Key: o.Key, Count: bc, category: o.category}
threshold = bc
} else {
results[end+1] = Pair{Key: o.Key, Count: bc, category: o.category}
sort.Sort(Pairs(results))
threshold = results[end].Count
}
}
}
return f.allocHandle(result)
return results[:end]
}
func (f *Fragment) Difference(bitmaps []BitmapHandle) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
/*
func (f *Fragment) TopFill(args FillArgs) ([]Pair, error) {
result := make([]Pair, 0)
for _, id := range args.Bitmaps {
if _, ok := f.cache.Get(id); !ok {
continue
}
result := NewBitmap()
for i, id := range bitmaps {
bm, _ := f.bitmap(id)
if i == 0 {
result = bm
} else {
result = result.Difference(bm)
if args.Handle == 0 {
if bm := f.Bitmap(id); bm != nil && bm.Count() > 0 {
result = append(result, Pair{Key: id, Count: bm.Count()})
}
continue
}
res := f.Intersect([]uint64{args.Handle, id})
if res == nil {
continue
}
if bc := res.BitCount(); bc > 0 {
result = append(result, Pair{Key: id, Count: bc})
}
}
return f.allocHandle(result)
return result, nil
}
*/
func (f *Fragment) Persist() {
func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap {
f.mu.Lock()
defer f.mu.Unlock()
err := f.impl.Persist()
if err != nil {
log.Warn("Error saving:", err)
bitmapIDs := GetRange(start, end, bitmapID)
if len(bitmapIDs) == 0 {
return NewBitmap()
}
}
func (f *Fragment) Load() {
f.mu.Lock()
defer f.mu.Unlock()
f.impl.Load(f)
}
// FragmentStats represents in-memory stats for a single fragment.
type FragmentStats struct {
// Messages processed by the fragment
ProcessN uint64
ProcessTime time.Duration
}
type Pilosa interface {
Get(id uint64) *Bitmap
SetBit(id uint64, bit_pos uint64, filter uint64) bool
ClearBit(id uint64, bit_pos uint64) bool
TopN(b *Bitmap, n int, categories []uint64) []Pair
TopNAll(n int, categories []uint64) []Pair
Clear() bool
Store(bitmapID uint64, bm *Bitmap, filter uint64) error
Stats() interface{}
Persist() error
Load(fragment *Fragment)
Exists(id uint64) bool
result := f.bitmap(bitmapIDs[0])
for _, id := range bitmapIDs[1:] {
result = result.Union(f.bitmap(id))
}
return result
}

340
fragment_container.go
View file

@ -1,340 +0,0 @@
package pilosa
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/gob"
"errors"
"fmt"
"io/ioutil"
"sync"
"syscall"
"time"
log "github.com/cihub/seelog"
// "github.com/umbel/pilosa/statsd"
)
// DefaultBackend is the default data storage layer.
const DefaultBackend = "cassandra"
var Backend = DefaultBackend
var LevelDBPath string
var (
ErrFragmentNotFound = errors.New("fragment not found")
)
func init() {
gob.Register(BitmapHandle(0))
gob.Register([]Pair{})
}
type FragmentContainer struct {
mu sync.Mutex
fragments map[SUUID]*Fragment
}
func NewFragmentContainer() *FragmentContainer {
return &FragmentContainer{
fragments: make(map[SUUID]*Fragment),
}
}
type BitmapHandle uint64
type FillArgs struct {
FragmentID SUUID
Handle BitmapHandle
Bitmaps []uint64
}
func (fc *FragmentContainer) Fragment(fragmentID SUUID) (*Fragment, bool) {
fc.mu.Lock()
f, ok := fc.fragments[fragmentID]
fc.mu.Unlock()
return f, ok
}
func (fc *FragmentContainer) Get(fragmentID SUUID, bitmapID uint64) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.NewHandle(bitmapID), nil
}
func (fc *FragmentContainer) LoadBitmap(fragmentID SUUID, bitmapID uint64, data string, filter uint64) error {
f, ok := fc.Fragment(fragmentID)
if !ok {
return ErrFragmentNotFound
}
// Decode from base64 encoding.
buf, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return err
}
// Decompress data.
reader, err := gzip.NewReader(bytes.NewReader(buf))
if err != nil {
return err
}
b, err := ioutil.ReadAll(reader)
if err != nil {
return err
}
// Build bitmap from data.
bm := NewBitmap()
bm.FromBytes(b)
// Write bitmap to the underlying store.
return f.impl.Store(bitmapID, bm, filter)
}
func (fc *FragmentContainer) Stats(fragmentID SUUID) interface{} {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil
}
return f.impl.Stats()
}
func (fc *FragmentContainer) Empty(fragmentID SUUID) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.AllocHandle(NewBitmap()), nil
}
func (fc *FragmentContainer) Intersect(fragmentID SUUID, bh []BitmapHandle) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.Intersect(bh), nil
}
func (fc *FragmentContainer) Union(fragmentID SUUID, bh []BitmapHandle) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.Union(bh), nil
}
func (fc *FragmentContainer) Difference(fragmentID SUUID, bh []BitmapHandle) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.Difference(bh), nil
}
func (fc *FragmentContainer) Mask(fragmentID SUUID, start, end uint64) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
bm := NewBitmap()
for i := start; i < end; i++ {
bm.SetBit(i)
}
return f.AllocHandle(bm), nil
}
func (fc *FragmentContainer) Range(fragmentID SUUID, bitmapID uint64, start, end time.Time) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.build_time_range_bitmap(bitmapID, start, end), nil
}
func (fc *FragmentContainer) TopN(fragmentID SUUID, bh BitmapHandle, n int, categories []uint64) ([]Pair, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
return f.TopN(bh, n, categories), nil
}
func (fc *FragmentContainer) TopNAll(fragmentID SUUID, n int, categories []uint64) ([]Pair, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
return f.TopNAll(n, categories), nil
}
func (fc *FragmentContainer) TopFillBatch(args []FillArgs) ([]Pair, error) {
results := make(map[uint64]uint64)
for _, v := range args {
items, _ := fc.TopFillFragment(v)
if len(args) == 1 {
return items, nil
}
for _, pair := range items {
results[pair.Key] += pair.Count
}
}
ret_val := make([]Pair, len(results))
for k, v := range results {
if v > 0 { //don't include 0 size items
ret_val = append(ret_val, Pair{k, v})
}
}
return ret_val, nil
}
func (fc *FragmentContainer) TopFillFragment(args FillArgs) ([]Pair, error) {
f, ok := fc.Fragment(args.FragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
result := make([]Pair, 0)
for _, v := range args.Bitmaps {
if !f.exists(v) {
continue
}
a := f.NewHandle(v)
if args.Handle == 0 {
// Return just the count
if bm, ok := f.Bitmap(a); ok && bm.Count() > 0 {
result = append(result, Pair{v, bm.Count()})
}
continue
}
res := f.Intersect([]BitmapHandle{args.Handle, a})
if bm, ok := f.Bitmap(res); ok {
bc := bm.BitCount()
if bc > 0 {
result = append(result, Pair{v, bc})
}
}
}
return result, nil
}
func (fc *FragmentContainer) GetList(fragmentID SUUID, bitmapIDs []uint64) ([]BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
a := make([]BitmapHandle, len(bitmapIDs))
for i, v := range bitmapIDs {
a[i] = f.NewHandle(v)
}
return a, nil
}
func (fc *FragmentContainer) Count(fragmentID SUUID, bh BitmapHandle) (uint64, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
bm, ok := f.Bitmap(bh)
if ok == false {
return 0, nil
}
return bm.BitCount(), nil
}
func (fc *FragmentContainer) GetBytes(fragmentID SUUID, bh BitmapHandle) ([]byte, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
bm, ok := f.Bitmap(bh)
if !ok {
bm = NewBitmap()
log.Warn("cache miss")
}
var b bytes.Buffer
w := gzip.NewWriter(&b)
if _, err := w.Write(bm.ToBytes()); err != nil {
return nil, err
}
if err := w.Flush(); err != nil {
return nil, err
}
if err := w.Close(); err != nil {
return nil, err
}
return b.Bytes(), nil
}
func (fc *FragmentContainer) FromBytes(fragmentID SUUID, data []byte) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
r, _ := gzip.NewReader(bytes.NewReader(data))
b, _ := ioutil.ReadAll(r)
bm := NewBitmap()
bm.FromBytes(b)
return f.AllocHandle(bm), nil
}
func (fc *FragmentContainer) SetBit(fragmentID SUUID, bitmapID uint64, pos uint64, category uint64) (bool, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return false, ErrFragmentNotFound
}
return f.impl.SetBit(bitmapID, pos, category), nil
}
func (fc *FragmentContainer) ClearBit(fragmentID SUUID, bitmapID uint64, pos uint64) (bool, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return false, ErrFragmentNotFound
}
return f.impl.ClearBit(bitmapID, pos), nil
}
func (fc *FragmentContainer) Clear(fragmentID SUUID) (bool, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return false, ErrFragmentNotFound
}
return f.impl.Clear(), nil
}
func (fc *FragmentContainer) AddFragment(db string, frame string, slice int, id SUUID) {
fc.mu.Lock()
defer fc.mu.Unlock()
_, ok := fc.fragments[id]
if ok {
return
}
log.Warn("ADD FRAGMENT", frame, db, slice, id.String())
f := NewFragment(id, db, slice, frame)
fc.fragments[id] = f
// go f.Load(loader)
}
func dumpHandlesToLog() {
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
log.Warn("Getrlimit:" + err.Error())
}
mesg := fmt.Sprintf("%v file descriptors out of a maximum of %v available\n", limit.Cur, limit.Max)
log.Warn(mesg)
}

308
fragment_container_test.go
View file

@ -1,308 +0,0 @@
package pilosa_test
import (
"testing"
"github.com/umbel/pilosa"
_ "github.com/umbel/pilosa/storage"
)
func init() {
pilosa.Backend = "memory"
}
// Ensure a fragment can be retrieved from the container.
func TestFragmentContainer_Get(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustClear(1)
if bh, err := fc.Get(pilosa.SUUID(1), 1234); err != nil {
t.Fatal(err)
} else if bh == 0 {
t.Fatal("expected non-zero bitmap handle")
}
}
// Ensure a bit can be set on a bitmap.
func TestFragmentContainer_SetBit(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustClear(1)
// Set a bit on the bitmap.
if changed, err := fc.SetBit(1, uint64(1234), 1, 0); err != nil {
t.Fatal(err)
} else if changed == false {
t.Fatal("expected change")
}
// Set the same bit on the bitmap. No change should be indicated.
if changed, err := fc.SetBit(1, uint64(1234), 1, 0); err != nil {
t.Fatal(err)
} else if changed == true {
t.Fatal("expected no change")
}
}
// Ensure the number of bits on a bitmap can be counted.
func TestFragmentContainer_Count(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, pilosa.SUUID(1))
// Set a bit on the bitmap.
bi1 := uint64(1234)
if changed, err := fc.SetBit(pilosa.SUUID(1), bi1, 1, 0); err != nil {
t.Fatal(err)
} else if changed == false {
t.Fatal("expected change")
}
// Verify that one bit is set.
if bh, err := fc.Get(pilosa.SUUID(1), bi1); err != nil {
t.Fatal(err)
} else if n, err := fc.Count(pilosa.SUUID(1), bh); err != nil {
t.Fatal(err)
} else if n != 1 {
t.Fatalf("unexpected count: %d", n)
}
}
// Ensure the bits in two bitmaps can be unioned.
func TestFragmentContainer_Union(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
fc.MustSetBit(1, 4321, 65537, 0)
// Union the handles together.
if result, err := fc.Union(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, result); n != 2 {
t.Fatalf("unexpected union bit count: %d", n)
}
}
// Ensure unioning a bitmap with an empty bitmap returns a single bit count.
func TestFragmentContainer_Union_Empty(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
// Union the handles together.
if result, err := fc.Union(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, result); n != 1 {
t.Fatalf("unexpected empty union bit count: %d", n)
}
}
// Ensure the bits in two bitmaps can be intersected.
func TestFragmentContainer_Intersect(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
fc.MustSetBit(1, 4321, 65537, 0)
// Intersect the handles together.
if result, err := fc.Intersect(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, result); n != 0 {
t.Fatalf("unexpected intersect bit count: %d", n)
}
}
// Ensure the bits in two bitmaps can be diffed.
func TestFragmentContainer_Difference(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
fc.MustSetBit(1, 4321, 65537, 0)
// Compute the difference between the handles.
if result, err := fc.Difference(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, result); n != 1 {
t.Fatalf("unexpected difference bit count: %s", err)
}
}
// Ensure bitmaps can be marshaled and unmarshaled to bytes.
func TestFragmentContainer_Bytes(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
// Count bits and marshal to bytes.
beforeN := fc.MustCount(1, 1234)
buf, err := fc.GetBytes(1, 1234)
if err != nil {
t.Fatal(err)
}
// Marshal bytes back to a bitmap and re-count.
bh2, err := fc.FromBytes(1, buf)
if err != nil {
t.Fatal(err)
}
afterN := fc.MustCount(1, bh2)
// Ensure the original bit count matches the new bitmap's bit count.
if beforeN != afterN {
t.Fatalf("unexpected bit count: before=%d, after=%d", beforeN, afterN)
}
}
// Ensure an empty bitmap can be returned.
func TestFragmentContainer_Empty(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
bh, err := fc.Empty(1)
if err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, bh); n != 0 {
t.Fatalf("unexpected bit count: %d", n)
}
}
// Ensure a list of bitmap handles can be returned.
func TestFragmentContainer_GetList(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
fc.MustSetBit(1, 4321, 65537, 0)
a, err := fc.GetList(1, []uint64{1234, 4321, 789})
if err != nil {
t.Fatal(err)
}
// Compute the union to ensure they're the correct bitmaps.
if res, err := fc.Union(1, a); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, res); n != 2 {
t.Fatalf("unexpected bit count: %d", n)
}
}
// Ensure brand bitmaps can perform a small number of set bits.
func TestFragmentContainer_SetBit_Brand_Small(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "b.n", 0, 2)
for i := uint64(0); i < 1000; i++ {
fc.SetBit(2, 1029, i, 0)
}
}
// Ensure the top n can be computed for a brand.
func TestFragmentContainer_TopN_Brand(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "b.n", 0, 2)
// Set bits on the bitmap.
fc.MustSetBit(2, uint64(1), 1, 2)
fc.MustSetBit(2, uint64(1), 2, 2)
fc.MustSetBit(2, uint64(1), 3, 2)
fc.MustSetBit(2, uint64(2), 1, 2)
fc.MustSetBit(2, uint64(2), 2, 2)
fc.MustSetBit(2, uint64(3), 1, 2)
// Retrieve the bitmap handle for bitmap 1.
bh := fc.MustGet(2, uint64(1))
// Compute the top-n.
if results, err := fc.TopN(2, bh, 4, []uint64{2}); err != nil {
t.Fatal(err)
} else if results[0].Key != 1 {
t.Fatalf("unexpected key: %d", results[0].Key)
} else if results[0].Count != 3 {
t.Fatalf("unexpected value: %d", results[0].Count)
}
}
// Ensure a fragment can be cleared.
func TestFragmentContainer_Clear(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
// Compute the top-n.
if res, err := fc.Clear(1); err != nil {
t.Fatal(err)
} else if res != true {
t.Fatalf("unexpected result: %v", res)
}
}
// Ensure a fragment can be marshaled and unmarshaled to and from bytes.
func TestFragmentContainer_FromBytes(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "b.n", 0, 2)
// Set bits for a bitmap.
for i := 0; i < 4096; i++ {
fc.MustSetBit(2, 1, uint64(i), 0)
}
// Marshal to bytes.
buf, err := fc.GetBytes(2, fc.MustGet(2, 1))
if err != nil {
t.Fatal(err)
}
// Load a bitmap from compressed data.
bh, err := fc.FromBytes(2, buf)
if err != nil {
t.Fatal(err)
}
// Load and count bits.
if n := fc.MustCount(2, bh); n != 4096 {
t.Fatalf("unexpected bit count: %d", n)
}
}
// FragementContainer is a test wrapper for pilosa.FragmentContainer.
type FragmentContainer struct {
*pilosa.FragmentContainer
}
// NewFragmentContainer returns a new instance of FragmentContainer.
func NewFragmentContainer() *FragmentContainer {
return &FragmentContainer{pilosa.NewFragmentContainer()}
}
// MustGet retrieves a bitmap by id. Panic on error.
func (fc *FragmentContainer) MustGet(frag_id pilosa.SUUID, bitmap_id uint64) pilosa.BitmapHandle {
bh, err := fc.Get(frag_id, bitmap_id)
if err != nil {
panic(err)
}
return bh
}
// MustSetBit sets a bit in a bitmap. Panic on error.
func (fc *FragmentContainer) MustSetBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64, category uint64) bool {
changed, err := fc.SetBit(frag_id, bitmap_id, pos, category)
if err != nil {
panic(err)
}
return changed
}
// MustClear clears a fragment. Panic on error.
func (fc *FragmentContainer) MustClear(fragmentID pilosa.SUUID) bool {
v, err := fc.Clear(fragmentID)
if err != nil {
panic(err)
}
return v
}
// MustCount returns the number of set bits in a bitmap. Panic on error.
func (fc *FragmentContainer) MustCount(frag_id pilosa.SUUID, bitmap pilosa.BitmapHandle) uint64 {
v, err := fc.Count(frag_id, bitmap)
if err != nil {
panic(err)
}
return v
}

8
fragment_test.go Normal file
View file

@ -0,0 +1,8 @@
package pilosa_test
import (
"github.com/umbel/pilosa"
)
// SliceWidth is a helper reference to use when testing.
const SliceWidth = pilosa.SliceWidth

174
general.go
View file

@ -1,174 +0,0 @@
package pilosa
import (
"encoding/json"
"fmt"
log "github.com/cihub/seelog"
"github.com/golang/groupcache/lru"
)
type General struct {
bitmap_cache *lru.Cache
keys map[uint64]interface{}
db string
frame string
slice int
storage Storage
}
func NewGeneral(db string, frame string, slice int, s Storage) *General {
f := new(General)
f.storage = s
f.frame = frame
f.slice = slice
f.db = db
f.Clear()
f.keys = make(map[uint64]interface{})
return f
}
func (self *General) Clear() bool {
self.bitmap_cache = lru.New(50000)
self.bitmap_cache.OnEvicted = self.OnEvicted
return true
}
func (self *General) Exists(bitmap_id uint64) bool {
_, ok := self.bitmap_cache.Get(bitmap_id)
return ok
}
func (self *General) Get(bitmap_id uint64) *Bitmap {
bm, ok := self.bitmap_cache.Get(bitmap_id)
if ok && bm != nil {
return bm.(*Bitmap)
}
bm, _ = self.storage.Fetch(bitmap_id, self.db, self.frame, self.slice)
self.bitmap_cache.Add(bitmap_id, bm)
self.keys[bitmap_id] = 0
return bm.(*Bitmap)
}
func (self *General) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool {
bm := self.Get(bitmap_id)
change, chunk, address := bm.SetBit(bit_pos)
if change {
val := chunk.Value[address.BlockIndex]
self.storage.StoreBit(bitmap_id, self.db, self.frame, self.slice, filter, address.ChunkKey, int32(address.BlockIndex), val, bm.Count())
}
return change
}
func (self *General) TopN(b *Bitmap, n int, categories []uint64) []Pair {
var empty []Pair
return empty
}
func (self *General) Store(bitmap_id uint64, bm *Bitmap, filter uint64) error {
if err := self.storage.Store(bitmap_id, self.db, self.frame, self.slice, filter, bm); err != nil {
return err
}
self.bitmap_cache.Add(bitmap_id, bm)
self.keys[bitmap_id] = 0
return nil
}
func (self *General) OnEvicted(key lru.Key, value interface{}) {
delete(self.keys, key.(uint64))
}
func (self *General) Stats() interface{} {
stats := map[string]interface{}{
"total size of cache in items": self.bitmap_cache.Len()}
return stats
}
func (self *General) getFileName() string {
base := FragmentBase
if base == "" {
base = "."
}
return fmt.Sprintf("%s/%s.%s.%d", base, self.db, self.frame, self.slice)
}
func (self *General) Persist() error {
log.Warn("General Persist")
w, err := createFile(self.getFileName())
if err != nil {
log.Warn("Error saving:", err)
return err
}
defer w.Close()
defer self.storage.Close()
results := make([]uint64, len(self.keys))
i := 0
for k, _ := range self.keys { // map[uint64]*Rank
results[i] = k
i += 1
}
encoder := json.NewEncoder(w)
return encoder.Encode(results)
}
func (self *General) Load(f *Fragment) {
log.Warn("General Load")
r, err := openFile(self.getFileName())
if err != nil {
log.Warn("NO General Init File:", self.getFileName())
return
}
dec := json.NewDecoder(r)
var keys []uint64
if err := dec.Decode(&keys); err != nil {
return
}
for _, k := range keys {
self.Get(k)
}
}
func (self *General) TopNAll(n int, categories []uint64) []Pair {
results := make([]Pair, 0, 0)
count := 0
for k, _ := range self.keys {
if count >= n {
break
}
bm := self.Get(k)
results = append(results, Pair{k, bm.Count()})
count++
}
return results
}
func (self *General) ClearBit(bitmap_id uint64, bit_pos uint64) bool {
bm := self.Get_nocache(bitmap_id)
if bm.Count() == 0 {
return false
}
change, chunk, address := bm.ClearBit(bit_pos)
if change {
val := chunk.Value[address.BlockIndex]
if val == 0 {
self.storage.RemoveBit(bitmap_id, self.db, self.frame, self.slice, uint64(0), address.ChunkKey, int32(address.BlockIndex), bm.Count())
} else {
self.storage.StoreBit(bitmap_id, self.db, self.frame, self.slice, uint64(0), address.ChunkKey, int32(address.BlockIndex), val, bm.Count())
}
}
return change
}
func (self *General) Get_nocache(bitmap_id uint64) *Bitmap {
bm, ok := self.bitmap_cache.Get(bitmap_id)
if ok && bm != nil {
return bm.(*Bitmap)
}
bm, _ = self.storage.Fetch(bitmap_id, self.db, self.frame, self.slice)
return bm.(*Bitmap)
}

261
handler.go Normal file
View file

@ -0,0 +1,261 @@
package pilosa
import (
"encoding/json"
"errors"
"expvar"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"strings"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa/internal"
"github.com/umbel/pilosa/pql"
)
// Handler represents an HTTP handler.
type Handler struct {
// The execution engine for running queries.
Executor interface {
Execute(db string, query *pql.Query, slices []uint64) (interface{}, error)
}
// The version to report on the /version endpoint.
Version string
// The writer for any logging.
LogOutput io.Writer
}
// NewHandler returns a new instance of Handler with a default logger.
func NewHandler() *Handler {
return &Handler{
Version: Version,
LogOutput: os.Stderr,
}
}
// ServeHTTP handles an HTTP request.
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/query":
h.handleQuery(w, r)
case "/version":
h.handleVersion(w, r)
case "/debug/vars":
h.handleExpvar(w, r)
default:
http.NotFound(w, r)
}
}
// handleQuery handles /query requests.
func (h *Handler) handleQuery(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "POST":
h.handlePostQuery(w, r)
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
// handlePostQuery handles /query requests.
func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
// Parse incoming request.
db, query, slices, err := h.readQueryRequest(r)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
h.writeQueryResponse(w, r, nil, err)
return
}
// Parse query string.
q, err := pql.NewParser(strings.NewReader(query)).Parse()
if err != nil {
w.WriteHeader(http.StatusBadRequest)
h.writeQueryResponse(w, r, nil, err)
return
}
// Execute the query.
res, e := h.Executor.Execute(db, q, slices)
// Set appropriate status code, if there is an error.
if e != nil {
w.WriteHeader(http.StatusInternalServerError)
}
// Write response back to client.
if err := h.writeQueryResponse(w, r, res, e); err != nil {
h.logger().Printf("write query response error: %s", err)
}
}
// readQueryRequest parses an query parameters from r.
func (h *Handler) readQueryRequest(r *http.Request) (db, query string, slices []uint64, err error) {
switch r.Header.Get("Content-Type") {
case "application/x-protobuf":
return h.readProtobufQueryRequest(r)
default:
return h.readURLQueryRequest(r)
}
}
// readProtobufQueryRequest parses query parameters in protobuf from r.
func (h *Handler) readProtobufQueryRequest(r *http.Request) (db, query string, slices []uint64, err error) {
// Slurp the body.
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
// Unmarshal into object.
var req internal.QueryRequest
if err = proto.Unmarshal(body, &req); err != nil {
return
}
return req.GetDB(), req.GetQuery(), req.GetSlices(), nil
}
// readURLQueryRequest parses query parameters from URL parameters from r.
func (h *Handler) readURLQueryRequest(r *http.Request) (db, query string, slices []uint64, err error) {
q := r.URL.Query()
// Read DB argument.
db = q.Get("db")
// Parse query string.
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
query = string(buf)
// Parse list of slices.
slices, err = parseUint64Slice(q.Get("slices"))
if err != nil {
err = errors.New("invalid slice argument")
return
}
return
}
// writeQueryResponse writes the response from the executor to w.
func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res interface{}, err error) error {
if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") {
return h.writeProtobufQueryResponse(w, res, err)
}
return h.writeJSONQueryResponse(w, res, err)
}
// writeProtobufQueryResponse writes the response from the executor to w as protobuf.
func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, res interface{}, e error) error {
var resp internal.QueryResponse
// Set the result on the appropriate field.
if res != nil {
switch res := res.(type) {
case *Bitmap:
resp.Bitmap = encodeBitmap(res)
case Pairs:
resp.Pairs = encodePairs(res)
case uint64:
resp.N = proto.Uint64(res)
default:
panic(fmt.Sprintf("invalid query response type: %T", res))
}
}
// Set the error if there is one.
if e != nil {
resp.Err = proto.String(e.Error())
}
// Encode response.
buf, err := proto.Marshal(&resp)
if err != nil {
return err
}
// Write response back to client.
if _, err := w.Write(buf); err != nil {
return err
}
return nil
}
// writeJSONQueryResponse writes the response from the executor to w as JSON.
func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, res interface{}, e error) error {
var o struct {
Result interface{} `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
o.Result = res
if e != nil {
o.Error = e.Error()
}
// Otherwise marshal the result as JSON.
return json.NewEncoder(w).Encode(o)
}
// handleGetVersion handles /version requests.
func (h *Handler) handleVersion(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(struct {
Version string `json:"version"`
}{
Version: h.Version,
}); err != nil {
h.logger().Printf("write version response error: %s", err)
}
}
// handleExpvar handles /debug/vars requests.
func (h *Handler) handleExpvar(w http.ResponseWriter, r *http.Request) {
// Copied from $GOROOT/src/expvar/expvar.go
w.Header().Set("Content-Type", "application/json; charset=utf-8")
fmt.Fprintf(w, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprintf(w, ",\n")
}
first = false
fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
})
fmt.Fprintf(w, "\n}\n")
}
// logger returns a logger for the handler.
func (h *Handler) logger() *log.Logger {
return log.New(h.LogOutput, "", log.LstdFlags)
}
// parseUint64Slice returns a slice of uint64s from a comma-delimited string.
func parseUint64Slice(s string) ([]uint64, error) {
var a []uint64
for _, str := range strings.Split(s, ",") {
// Ignore blanks.
if str == "" {
continue
}
// Parse number.
num, err := strconv.ParseUint(str, 10, 64)
if err != nil {
return nil, err
}
a = append(a, num)
}
return a, nil
}

373
handler_test.go Normal file
View file

@ -0,0 +1,373 @@
package pilosa_test
import (
"bytes"
"errors"
"io"
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"strings"
"testing"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/internal"
"github.com/umbel/pilosa/pql"
)
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_NotFound(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/no_such_path", nil))
if w.Code != http.StatusNotFound {
t.Fatalf("invalid status: %d", w.Code)
}
}
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `count(get(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return uint64(100), nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("count( get( 100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":100}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can accept arguments via protobufs.
func TestHandler_Query_Args_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `count(get(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return uint64(100), nil
}
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
DB: proto.String("db0"),
Query: proto.String("count(get(100))"),
Slices: []uint64{0, 1},
})
if err != nil {
t.Fatal(err)
}
// Generate protobuf request.
req := MustNewHTTPRequest("POST", "/query", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/x-protobuf")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Ensure the handler returns an error when parsing bad arguments.
func TestHandler_Query_Args_Err(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=a,b", strings.NewReader("get(100)")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query with a uint64 response as JSON.
func TestHandler_Query_Uint64_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return uint64(100), nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("count( get( 100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":100}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query with a uint64 response as protobufs.
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return uint64(100), nil
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader("count(get(100))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if resp.GetN() != 100 {
t.Fatalf("unexpected n: %d", resp.GetN())
}
}
// Ensure the handler can execute a query that returns a bitmap as JSON.
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
bm := pilosa.NewBitmap()
bm.SetBit(1)
bm.SetBit(3)
bm.SetBit(66)
bm.SetBit(pilosa.SliceWidth + 1)
return bm, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader("get(100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":{"chunks":[{"Key":0,"Value":[10,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},{"Key":32,"Value":[2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}]}}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query that returns a bitmap as protobuf.
func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
bm := pilosa.NewBitmap()
bm.SetBit(1)
bm.SetBit(pilosa.SliceWidth + 1)
return bm, nil
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader("get(100)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.GetBitmap().GetChunks(); len(a) != 2 {
t.Fatalf("unexpected bitmap chunk length: %d", len(a))
}
}
// Ensure the handler can execute a query that returns pairs as JSON.
func TestHandler_Query_Pairs_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return pilosa.Pairs{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`top-n(frame=x, n=2)`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":[{"key":1,"count":2},{"key":3,"count":4}]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query that returns pairs as protobuf.
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return pilosa.Pairs{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader(`top-n(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
}
// Ensure the handler can return an error as JSON.
func TestHandler_Query_Err_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`get(100)`)))
if w.Code != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can return an error as protobuf.
func TestHandler_Query_Err_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader(`top-n(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if s := resp.GetErr(); s != `marker` {
t.Fatalf("unexpected error: %s", s)
}
}
// Ensure the handler returns "method not allowed" for non-POST queries.
func TestHandler_Query_MethodNotAllowed(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/query", nil))
if w.Code != http.StatusMethodNotAllowed {
t.Fatalf("invalid status: %d", w.Code)
}
}
// Ensure the handler returns an error if there is a parsing error..
func TestHandler_Query_ErrParse(t *testing.T) {
h := NewHandler()
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"function not found: bad_fn occurred at line 1, char 1"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can retrieve the version.
func TestHandler_Version(t *testing.T) {
h := NewHandler()
h.Version = "1.0.0"
w := httptest.NewRecorder()
r := MustNewHTTPRequest("GET", "/version", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"version":"1.0.0"}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// Ensure the handler can return expvars without panicking.
func TestHandler_Expvars(t *testing.T) {
h := NewHandler()
w := httptest.NewRecorder()
r := MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Handler represents a test wrapper for pilosa.Handler.
type Handler struct {
*pilosa.Handler
Executor HandlerExecutor
}
// NewHandler returns a new instance of Handler.
func NewHandler() *Handler {
h := &Handler{
Handler: pilosa.NewHandler(),
}
h.Handler.Executor = &h.Executor
return h
}
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(db string, query *pql.Query, slices []uint64) (interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return c.ExecuteFn(db, query, slices)
}
// Server represents a test wrapper for httptest.Server.
type Server struct {
*httptest.Server
Handler *Handler
}
// NewServer returns a test server running on a random port.
func NewServer() *Server {
s := &Server{
Handler: NewHandler(),
}
s.Server = httptest.NewServer(s.Handler.Handler)
return s
}
// Host returns the hostname of the running server.
func (s *Server) Host() string {
u, err := url.Parse(s.URL)
if err != nil {
panic(err)
}
return u.Host
}
// MustNewHTTPRequest creates a new HTTP request. Panic on error.
func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request {
req, err := http.NewRequest(method, urlStr, body)
if err != nil {
panic(err)
}
return req
}

88
hold/hold.go
View file

@ -1,88 +0,0 @@
package hold
import (
"errors"
"time"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
)
type holdchan chan interface{}
type gethold struct {
id *pilosa.GUID
reply chan holdchan
}
type delhold struct {
id *pilosa.GUID
}
type Holder struct {
data map[pilosa.GUID]holdchan
getchan chan gethold
delchan chan delhold
}
func NewHolder() *Holder {
return &Holder{
data: make(map[pilosa.GUID]holdchan),
getchan: make(chan gethold),
delchan: make(chan delhold),
}
}
func (self *Holder) DelChan(id *pilosa.GUID) {
log.Trace("Holder.DelChan", id)
req := delhold{id}
self.delchan <- req
}
func (self *Holder) GetChan(id *pilosa.GUID) holdchan {
log.Trace("Holder.GetChan", id)
reply := make(chan holdchan)
req := gethold{id, reply}
self.getchan <- req
return <-reply
}
func (self *Holder) Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error) {
log.Trace("Holder.Get", id, timeout.String())
ch := self.GetChan(id)
select {
case val := <-ch:
return val, nil
case <-time.After(timeout):
self.DelChan(id)
return nil, errors.New("Timeout getting from holder")
}
}
func (self *Holder) Set(id *pilosa.GUID, value interface{}, timeout time.Duration) {
log.Trace("Holder.Set", id, value, timeout.String())
ch := self.GetChan(id)
go func() {
select {
case ch <- value:
case <-time.After(timeout):
}
self.DelChan(id)
}()
}
func (self *Holder) Run() {
var greq gethold
var dreq delhold
for {
select {
case greq = <-self.getchan:
item, ok := self.data[*greq.id]
if !ok {
item = make(holdchan)
self.data[*greq.id] = item
}
greq.reply <- item
case dreq = <-self.delchan:
delete(self.data, *dreq.id)
}
}
}

41
hold/hold_test.go
View file

@ -1,41 +0,0 @@
package hold_test
import (
"testing"
"time"
"github.com/umbel/pilosa/hold"
"github.com/umbel/pilosa/util"
)
// Ensure hold can get a value that has been set.
func TestHold_Get(t *testing.T) {
h := hold.NewHolder()
go h.Run()
id := util.RandomUUID()
h.Set(&id, "derp", 10)
if v, err := h.Get(&id, 10); err != nil {
t.Fatal(err)
} else if v != "derp" {
t.Fatalf("unexpected value: %v", v)
}
}
// Ensure hold can wait for a value that has not been set yet.
func TestHold_Get_Delay(t *testing.T) {
h := hold.NewHolder()
go h.Run()
id := util.RandomUUID()
go func() {
time.Sleep(500 * time.Millisecond)
h.Set(&id, "derpsy", 10)
}()
if v, err := h.Get(&id, 10); err != nil {
t.Fatal(err)
} else if v != "derpsy" {
t.Fatalf("unexpected value: %v", v)
}
}

53
index.go Normal file
View file

@ -0,0 +1,53 @@
package pilosa
import (
"sync"
)
// Index represents a container for fragments.
type Index struct {
mu sync.Mutex
sliceN uint64
fragments map[fragmentKey]*Fragment
}
// NewIndex returns a new instance of Index.
func NewIndex() *Index {
return &Index{
fragments: make(map[fragmentKey]*Fragment),
}
}
// SliceN returs the total number of slices managed by the index.
func (i *Index) SliceN() uint64 {
i.mu.Lock()
defer i.mu.Unlock()
return i.sliceN
}
// Fragment returns the fragment for a database, frame & slice.
// The fragment is created if it doesn't already exist.
func (i *Index) Fragment(db, frame string, slice uint64) *Fragment {
i.mu.Lock()
defer i.mu.Unlock()
// Track the highest slice.
if slice > i.sliceN {
i.sliceN = slice
}
// Create fragment, if not exists.
key := fragmentKey{db, frame, slice}
if i.fragments[key] == nil {
i.fragments[key] = NewFragment(db, frame, slice)
}
return i.fragments[key]
}
// fragmentKey is the map key for fragment look ups.
type fragmentKey struct {
db string
frame string
slice uint64
}

56
internal/internal.go Normal file
View file

@ -0,0 +1,56 @@
package internal
import (
"io"
"io/ioutil"
"github.com/gogo/protobuf/proto"
)
type Request proto.Message
type Response proto.Message
// Encoder encodes messages to a writer.
type Encoder struct {
w io.Writer
}
// NewEncoder returns a new instance of Encoder.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{w: w}
}
// Encode marshals m into bytes and writes them to r.
func (enc *Encoder) Encode(pb proto.Message) error {
buf, err := proto.Marshal(pb)
if err != nil {
return err
}
if _, err := enc.w.Write(buf); err != nil {
return err
}
return nil
}
// Decoder decodes messages from a reader.
type Decoder struct {
r io.Reader
}
// NewDecoder returns a new instance of Decoder.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: r}
}
// Decode reads all bytes from the reader and unmarshals them into pb.
func (dec *Decoder) Decode(pb proto.Message) error {
buf, err := ioutil.ReadAll(dec.r)
if err != nil {
return err
}
return proto.Unmarshal(buf, pb)
}

164
internal/internal.pb.go Normal file
View file

@ -0,0 +1,164 @@
// Code generated by protoc-gen-gogo.
// source: internal/internal.proto
// DO NOT EDIT!
/*
Package internal is a generated protocol buffer package.
It is generated from these files:
internal/internal.proto
It has these top-level messages:
Bitmap
Chunk
Pair
QueryRequest
QueryResponse
*/
package internal
import proto "github.com/gogo/protobuf/proto"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = math.Inf
type Bitmap struct {
Chunks []*Chunk `protobuf:"bytes,1,rep" json:"Chunks,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Bitmap) Reset() { *m = Bitmap{} }
func (m *Bitmap) String() string { return proto.CompactTextString(m) }
func (*Bitmap) ProtoMessage() {}
func (m *Bitmap) GetChunks() []*Chunk {
if m != nil {
return m.Chunks
}
return nil
}
type Chunk struct {
Key *uint64 `protobuf:"varint,1,req" json:"Key,omitempty"`
Value []uint64 `protobuf:"varint,2,rep" json:"Value,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Chunk) Reset() { *m = Chunk{} }
func (m *Chunk) String() string { return proto.CompactTextString(m) }
func (*Chunk) ProtoMessage() {}
func (m *Chunk) GetKey() uint64 {
if m != nil && m.Key != nil {
return *m.Key
}
return 0
}
func (m *Chunk) GetValue() []uint64 {
if m != nil {
return m.Value
}
return nil
}
type Pair struct {
Key *uint64 `protobuf:"varint,1,req" json:"Key,omitempty"`
Count *uint64 `protobuf:"varint,2,req" json:"Count,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Pair) Reset() { *m = Pair{} }
func (m *Pair) String() string { return proto.CompactTextString(m) }
func (*Pair) ProtoMessage() {}
func (m *Pair) GetKey() uint64 {
if m != nil && m.Key != nil {
return *m.Key
}
return 0
}
func (m *Pair) GetCount() uint64 {
if m != nil && m.Count != nil {
return *m.Count
}
return 0
}
type QueryRequest struct {
DB *string `protobuf:"bytes,1,req" json:"DB,omitempty"`
Query *string `protobuf:"bytes,2,req" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,3,rep" json:"Slices,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
func (m *QueryRequest) String() string { return proto.CompactTextString(m) }
func (*QueryRequest) ProtoMessage() {}
func (m *QueryRequest) GetDB() string {
if m != nil && m.DB != nil {
return *m.DB
}
return ""
}
func (m *QueryRequest) GetQuery() string {
if m != nil && m.Query != nil {
return *m.Query
}
return ""
}
func (m *QueryRequest) GetSlices() []uint64 {
if m != nil {
return m.Slices
}
return nil
}
type QueryResponse struct {
Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"`
Bitmap *Bitmap `protobuf:"bytes,2,opt" json:"Bitmap,omitempty"`
N *uint64 `protobuf:"varint,3,opt" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,4,rep" json:"Pairs,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *QueryResponse) Reset() { *m = QueryResponse{} }
func (m *QueryResponse) String() string { return proto.CompactTextString(m) }
func (*QueryResponse) ProtoMessage() {}
func (m *QueryResponse) GetErr() string {
if m != nil && m.Err != nil {
return *m.Err
}
return ""
}
func (m *QueryResponse) GetBitmap() *Bitmap {
if m != nil {
return m.Bitmap
}
return nil
}
func (m *QueryResponse) GetN() uint64 {
if m != nil && m.N != nil {
return *m.N
}
return 0
}
func (m *QueryResponse) GetPairs() []*Pair {
if m != nil {
return m.Pairs
}
return nil
}
func init() {
}

28
internal/internal.proto Normal file
View file

@ -0,0 +1,28 @@
package internal;
message Bitmap {
repeated Chunk Chunks = 1;
}
message Chunk {
required uint64 Key = 1;
repeated uint64 Value = 2;
}
message Pair {
required uint64 Key = 1;
required uint64 Count = 2;
}
message QueryRequest {
required string DB = 1;
required string Query = 2;
repeated uint64 Slices = 3;
}
message QueryResponse {
optional string Err = 1;
optional Bitmap Bitmap = 2;
optional uint64 N = 3;
repeated Pair Pairs = 4;
}

122
pilosa.go
View file

@ -1,122 +1,6 @@
package pilosa
import (
"encoding/binary"
"encoding/hex"
"fmt"
"math/rand"
"strings"
"time"
//go:generate protoc --gogo_out=. internal/internal.proto
"github.com/gocql/gocql"
)
var counter = uint64(0)
func init() {
rand.Seed(time.Now().UTC().UnixNano())
}
// SUUID represents a sequential UUID.
type SUUID uint64
// SUUID returns a new SUUID.
func NewSUUID() SUUID {
millis := uint64(time.Now().UTC().UnixNano())
id := millis << (64 - 41)
id |= uint64(rand.Intn(128)) << (64 - 41 - 13)
id |= counter % 1024
counter += 1
return SUUID(id)
}
// String returns a string representation of id.
func (id SUUID) String() string {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], uint64(id))
return hex.EncodeToString(buf[:])
}
// ParseSUUID parses s into an SUUID.
func ParseSUUID(s string) SUUID {
if n := len(s); n < 16 {
s = strings.Repeat("0", 16-n) + s
}
b, _ := hex.DecodeString(s)
return SUUID(binary.BigEndian.Uint64(b))
}
// GUID represents a globally unique identifier.
type GUID [16]byte
// UnmarshalText parses a text value into a GUID.
// This is used by the TOML parser.
func (id *GUID) UnmarshalText(text []byte) error {
v, err := ParseGUID(string(text))
if err != nil {
return err
}
*id = v
return nil
}
func (id GUID) String() string {
var offsets = [...]int{0, 2, 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 28, 30, 32, 34}
const hexString = "0123456789abcdef"
r := make([]byte, 36)
for i, b := range id {
r[offsets[i]] = hexString[b>>4]
r[offsets[i]+1] = hexString[b&0xF]
}
r[8] = '-'
r[13] = '-'
r[18] = '-'
r[23] = '-'
return string(r)
}
// Equals returns true if id equals other.
func (id *GUID) Equals(other *GUID) bool {
for i, v := range id {
if v != other[i] {
return false
}
}
return true
}
// NewGUID returns a random GUID.
func NewGUID() GUID {
uid, _ := gocql.RandomUUID()
var id GUID
copy(id[:], uid[:])
return id
}
// ParseGUID parses s into a GUID.
func ParseGUID(s string) (GUID, error) {
var u GUID
j := 0
for _, r := range s {
switch {
case r == '-' && j&1 == 0:
continue
case r >= '0' && r <= '9' && j < 32:
u[j/2] |= byte(r-'0') << uint(4-j&1*4)
case r >= 'a' && r <= 'f' && j < 32:
u[j/2] |= byte(r-'a'+10) << uint(4-j&1*4)
case r >= 'A' && r <= 'F' && j < 32:
u[j/2] |= byte(r-'A'+10) << uint(4-j&1*4)
default:
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
j += 1
}
if j != 32 {
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
return u, nil
}
// Version represents the current running version of Pilosa.
var Version string

51
pilosa_test.go
View file

@ -1,51 +0,0 @@
package pilosa_test
import (
"fmt"
"testing"
"github.com/umbel/pilosa"
)
// Ensure id can be parsed from string.
func TestSUUID_Small(t *testing.T) {
if v := pilosa.ParseSUUID("1"); v != 1 {
t.Fatalf("unexpected SUUID: %v", v)
}
}
// Ensure generated IDs are unique.
func TestSUUID_Unique(t *testing.T) {
a, b := pilosa.NewSUUID(), pilosa.NewSUUID()
if a == b {
t.Fatalf("ids should be unique: %v != %v", a, b)
}
}
// Ensure ids can be converted to and from hex.
func TestSUUID_Hex(t *testing.T) {
a := pilosa.NewSUUID()
b := pilosa.ParseSUUID(a.String())
if a != b {
t.Fatalf("ids not equal: %v != %v", a, b)
}
}
// Ensure ids can be generated in sequence.
func TestSUUID_Multiple(t *testing.T) {
for i := 0; i < 10; i++ {
println(pilosa.NewSUUID().String())
}
}
// Ensure a random GUID can be converted to a string.
func TestGUID_String(t *testing.T) {
fmt.Println(pilosa.NewGUID().String())
}
func BenchmarkSUUID(b *testing.B) {
// run the Fib function b.N times
for n := 0; n < b.N; n++ {
pilosa.NewSUUID()
}
}

24
popcnt_generic.go
View file

@ -2,22 +2,8 @@
package pilosa
func popcntSlice(s []uint64) uint64 {
return popcntSliceGo(s)
}
func popcntMaskSlice(s, m []uint64) uint64 {
return popcntMaskSliceGo(s, m)
}
func popcntAndSlice(s, m []uint64) uint64 {
return popcntAndSliceGo(s, m)
}
func popcntOrSlice(s, m []uint64) uint64 {
return popcntOrSliceGo(s, m)
}
func popcntXorSlice(s, m []uint64) uint64 {
return popcntXorSliceGo(s, m)
}
func popcntSlice(s []uint64) uint64 { return popcntSliceGo(s) }
func popcntMaskSlice(s, m []uint64) uint64 { return popcntMaskSliceGo(s, m) }
func popcntAndSlice(s, m []uint64) uint64 { return popcntAndSliceGo(s, m) }
func popcntOrSlice(s, m []uint64) uint64 { return popcntOrSliceGo(s, m) }
func popcntXorSlice(s, m []uint64) uint64 { return popcntXorSliceGo(s, m) }

228
pql/ast.go Normal file
View file

@ -0,0 +1,228 @@
package pql
import (
"fmt"
"strings"
"time"
)
// Query represents a PQL query.
type Query struct {
Root Call
}
// String returns a string representation of the query.
func (q *Query) String() string { return q.Root.String() }
// Node represents any node in the AST.
type Node interface {
node()
String() string
}
func (*Clear) node() {}
func (*Count) node() {}
func (*Difference) node() {}
func (*Get) node() {}
func (*Intersect) node() {}
func (*Range) node() {}
func (*Set) node() {}
func (*TopN) node() {}
func (*Union) node() {}
// Call represents a function call in the AST.
type Call interface {
Node
call()
}
func (*Clear) call() {}
func (*Count) call() {}
func (*Difference) call() {}
func (*Get) call() {}
func (*Intersect) call() {}
func (*Range) call() {}
func (*Set) call() {}
func (*TopN) call() {}
func (*Union) call() {}
// Calls represents a list of calls.
type Calls []Call
// String returns a string representation of the calls as a comma-delimited list.
func (a Calls) String() string {
args := make([]string, len(a))
for i, c := range a {
args[i] = c.String()
}
return strings.Join(args, ", ")
}
// BitmapCall represents a function call that returns a bitmap.
type BitmapCall interface {
Call
bitmapCall()
}
// BitmapCalls represents a list of bitmap calls.
type BitmapCalls []BitmapCall
// String returns a string representation of the calls as a comma-delimited list.
func (a BitmapCalls) String() string {
args := make([]string, len(a))
for i, c := range a {
args[i] = c.String()
}
return strings.Join(args, ", ")
}
func (*Difference) bitmapCall() {}
func (*Get) bitmapCall() {}
func (*Intersect) bitmapCall() {}
func (*Range) bitmapCall() {}
func (*Union) bitmapCall() {}
// Clear represents a clear() function call.
type Clear struct {
ID uint64
Frame string
Filter uint64
ProfileID uint64
}
// String returns the string representation of the call.
func (c *Clear) String() string {
args := make([]string, 0, 4)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.Filter != 0 {
args = append(args, fmt.Sprintf("filter=%d", c.Filter))
}
if c.ProfileID != 0 {
args = append(args, fmt.Sprintf("profile_id=%d", c.ProfileID))
}
return fmt.Sprintf("clear(%s)", strings.Join(args, ", "))
}
// Count represents a count() function call.
type Count struct {
Input BitmapCall
}
// String returns the string representation of the call.
func (c *Count) String() string {
return fmt.Sprintf("count(%s)", c.Input.String())
}
// Difference represents an difference() function call.
type Difference struct {
Inputs BitmapCalls
}
// String returns the string representation of the call.
func (c *Difference) String() string {
return fmt.Sprintf("difference(%s)", c.Inputs.String())
}
// Get represents a get() function call.
type Get struct {
ID uint64
Frame string
}
// String returns the string representation of the call.
func (c *Get) String() string {
args := make([]string, 0, 2)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
return fmt.Sprintf("get(%s)", strings.Join(args, ", "))
}
// Intersect represents an intersect() function call.
type Intersect struct {
Inputs BitmapCalls
}
// String returns the string representation of the call.
func (c *Intersect) String() string {
return fmt.Sprintf("intersect(%s)", c.Inputs.String())
}
// Range represents a range() function call.
type Range struct {
ID uint64
Frame string
StartTime time.Time
EndTime time.Time
}
// String returns the string representation of the call.
func (c *Range) String() string {
args := make([]string, 0, 2)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if !c.StartTime.IsZero() {
args = append(args, fmt.Sprintf("start=%s", c.StartTime.Format(TimeFormat)))
}
if !c.EndTime.IsZero() {
args = append(args, fmt.Sprintf("end=%s", c.EndTime.Format(TimeFormat)))
}
return fmt.Sprintf("range(%s)", strings.Join(args, ", "))
}
// Set represents a set() function call.
type Set struct {
ID uint64
Frame string
Filter uint64
ProfileID uint64
}
// String returns the string representation of the call.
func (c *Set) String() string {
args := make([]string, 0, 2)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.Filter != 0 {
args = append(args, fmt.Sprintf("filter=%d", c.Filter))
}
if c.ProfileID != 0 {
args = append(args, fmt.Sprintf("profile_id=%d", c.ProfileID))
}
return fmt.Sprintf("set(%s)", strings.Join(args, ", "))
}
// TopN represents a top-n() function call.
type TopN struct {
Frame string
N int
}
// String returns the string representation of the call.
func (c *TopN) String() string { panic("FIXME") }
// Union represents a union() function call.
type Union struct {
Inputs BitmapCalls
}
// String returns the string representation of the call.
func (c *Union) String() string {
return fmt.Sprintf("union(%s)", c.Inputs.String())
}

89
pql/ast_test.go Normal file
View file

@ -0,0 +1,89 @@
package pql_test
import (
"testing"
"time"
"github.com/umbel/pilosa/pql"
)
// Ensure the Clear call can be converted into a string.
func TestClear_String(t *testing.T) {
s := (&pql.Clear{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String()
if s != `clear(id=1, frame=x.n, filter=2, profile_id=3)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Count call can be converted into a string.
func TestCount_String(t *testing.T) {
s := (&pql.Count{Input: &pql.Get{ID: 1, Frame: "x.n"}}).String()
if s != `count(get(id=1, frame=x.n))` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Difference call can be converted into a string.
func TestDifference_String(t *testing.T) {
s := (&pql.Difference{Inputs: pql.BitmapCalls{
&pql.Get{ID: 1, Frame: "x.n"},
&pql.Get{ID: 2},
},
}).String()
if s != `difference(get(id=1, frame=x.n), get(id=2))` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Get call can be converted into a string.
func TestGet_String(t *testing.T) {
s := (&pql.Get{ID: 1, Frame: "x.n"}).String()
if s != `get(id=1, frame=x.n)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Intersect call can be converted into a string.
func TestIntersect_String(t *testing.T) {
s := (&pql.Intersect{Inputs: pql.BitmapCalls{
&pql.Get{ID: 1, Frame: "x.n"},
&pql.Get{ID: 2},
},
}).String()
if s != `intersect(get(id=1, frame=x.n), get(id=2))` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Range call can be converted into a string.
func TestRange_String(t *testing.T) {
s := (&pql.Range{
ID: 1,
Frame: "x.n",
StartTime: time.Unix(0, 0).UTC(),
EndTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
}).String()
if s != `range(id=1, frame=x.n, start=1970-01-01T00:00, end=2000-01-02T03:04)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Set call can be converted into a string.
func TestSet_String(t *testing.T) {
s := (&pql.Set{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String()
if s != `set(id=1, frame=x.n, filter=2, profile_id=3)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Union call can be converted into a string.
func TestUnion_String(t *testing.T) {
s := (&pql.Union{Inputs: pql.BitmapCalls{
&pql.Get{ID: 1, Frame: "x.n"},
&pql.Get{ID: 2},
},
}).String()
if s != `union(get(id=1, frame=x.n), get(id=2))` {
t.Fatalf("unexpected string: %s", s)
}
}

569
pql/parser.go Normal file
View file

@ -0,0 +1,569 @@
package pql
import (
"fmt"
"io"
"strconv"
"strings"
"time"
)
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"
// Parser represents a parser for the PQL language.
type Parser struct {
scanner *bufScanner
}
// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
return &Parser{
scanner: newBufScanner(r),
}
}
// ParseString parses s into a query.
func ParseString(s string) (*Query, error) {
return NewParser(strings.NewReader(s)).Parse()
}
// Parse parses the next node in the query.
func (p *Parser) Parse() (*Query, error) {
fn, err := p.parseCall()
if err != nil {
return nil, err
}
return &Query{Root: fn}, nil
}
// parseCall parses the next function call.
func (p *Parser) parseCall() (Call, error) {
tok, pos, lit := p.scanIgnoreWhitespace()
if tok != IDENT {
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
}
switch lit {
case "count":
return p.parseCountCall()
case "clear":
return p.parseClearCall()
case "difference":
return p.parseDifferenceCall()
case "get":
return p.parseGetCall()
case "intersect":
return p.parseIntersectCall()
case "range":
return p.parseRangeCall()
case "set":
return p.parseSetCall()
case "top-n":
return p.parseTopNCall()
case "union":
return p.parseUnionCall()
default:
return nil, &ParseError{Message: fmt.Sprintf("function not found: %s", lit), Pos: pos}
}
}
// parseClearCall parses a clear() function call.
func (p *Parser) parseClearCall() (*Clear, error) {
c := &Clear{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "filter":
if err := decodeUint64(arg.value, &c.Filter); err != nil {
return nil, parseErrorf(pos, "filter: %s", err)
}
case 3, "profile_id":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profile_id: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseCount parses a count() function call.
func (p *Parser) parseCountCall() (*Count, error) {
c := &Count{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
} else if len(args) != 1 {
return nil, parseErrorf(pos, "count requires one argument")
}
// Copy argument to AST.
input, ok := args[0].value.(BitmapCall)
if !ok {
return nil, parseErrorf(pos, "invalid count arg: %s", args[0].value)
}
c.Input = input
return c, nil
}
// parseDifference parses a difference() function call.
func (p *Parser) parseDifferenceCall() (*Difference, error) {
c := &Difference{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid arg: %v", arg.value)
}
}
return c, nil
}
// parseGetCall parses a get() function call.
func (p *Parser) parseGetCall() (*Get, error) {
c := &Get{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseIntersect parses a intersect() function call.
func (p *Parser) parseIntersectCall() (*Intersect, error) {
c := &Intersect{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid arg: %v", arg.value)
}
}
return c, nil
}
// parseRangeCall parses a range() function call.
func (p *Parser) parseRangeCall() (*Range, error) {
c := &Range{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "start: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "start":
if err := decodeDate(arg.value, &c.StartTime); err != nil {
return nil, parseErrorf(pos, "start: %s", err)
}
case 3, "end":
if err := decodeDate(arg.value, &c.EndTime); err != nil {
return nil, parseErrorf(pos, "end: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseSetCall parses a set() function call.
func (p *Parser) parseSetCall() (*Set, error) {
c := &Set{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "filter":
if err := decodeUint64(arg.value, &c.Filter); err != nil {
return nil, parseErrorf(pos, "filter: %s", err)
}
case 3, "profile_id":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profile_id: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseTopNCall parses a top-n() function call.
func (p *Parser) parseTopNCall() (*TopN, error) {
c := &TopN{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 1, "n":
if err := decodeInt(arg.value, &c.N); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseUnion parses a union() function call.
func (p *Parser) parseUnionCall() (*Union, error) {
c := &Union{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid arg: %v", arg.value)
}
}
return c, nil
}
// parseArgs arguments to a function call.
func (p *Parser) parseArgs() ([]arg, error) {
var i int
var args []arg
for {
// Parse next argument.
arg, err := p.parseArg()
if err != nil {
return nil, err
}
// If it's a primitive type without a key then index it.
if arg.key == nil {
switch arg.value.(type) {
case uint64, string:
arg.key = i
i++
}
}
// Append argument to list.
args = append(args, arg)
// If next token is a closing parenthesis, then exit.
// Otherwise expect a comma.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
break
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected COMMA, found %q", lit)
}
}
return args, nil
}
// parseArg parses a single argument to a function call.
func (p *Parser) parseArg() (arg, error) {
var key, value interface{}
// Read identifier and check if there's a following "=" or "(".
tok, pos, lit := p.scanIgnoreWhitespace()
switch tok {
case IDENT:
// If a left paren immediately follows then it's a function call.
if tok, _, _ := p.scan(); tok == LPAREN {
p.unscan(2)
c, err := p.parseCall()
if err != nil {
return arg{}, err
}
return arg{value: c}, nil
}
// If it's not a left paren, rescan ignoring whitespace and look for "=",
p.unscan(1)
if tok, _, _ := p.scanIgnoreWhitespace(); tok == EQ {
key = lit // keyed arg
} else {
p.unscan(1)
}
default:
p.unscan(1)
}
// Read value token.
tok, pos, lit = p.scanIgnoreWhitespace()
switch tok {
case IDENT, STRING:
value = lit
case NUMBER:
v, err := strconv.ParseUint(lit, 10, 64)
if err != nil {
return arg{}, err
}
value = v
case LBRACK:
panic("FIXME: parse list of integers")
default:
return arg{}, parseErrorf(pos, "invalid value: %q", lit)
}
return arg{key: key, value: value}, nil
}
// scan returns the next token from the scanner.
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() }
// scanIgnoreWhitespace returns the next non-whitespace token from the scanner.
func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) {
tok, pos, lit = p.scan()
if tok == WS {
tok, pos, lit = p.scan()
}
return
}
// unscan returns the last n tokens back to the scanner.
func (p *Parser) unscan(n int) {
for i := 0; i < n; i++ {
p.scanner.unscan()
}
}
// expect returns an error if the next token is not exp.
func (p *Parser) expect(exp Token) error {
if tok, pos, lit := p.scan(); tok != exp {
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
}
return nil
}
// expectIgnoreWhitespace returns an error if the next non-whitespace token is not exp.
func (p *Parser) expectIgnoreWhitespace(exp Token) error {
if tok, pos, lit := p.scanIgnoreWhitespace(); tok != exp {
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
}
return nil
}
// pos returns the current position.
func (p *Parser) pos() Pos { return p.scanner.pos() }
// arg represents an call argument.
// The key can be the index or the string key.
// The value can be a uint64, []uint64, string, Call, or Calls.
type arg struct {
key interface{}
value interface{}
}
// ParseError represents an error that occurred while parsing a PQL query.
type ParseError struct {
Message string
Pos Pos
}
// Error returns a string representation of e.
func (e *ParseError) Error() string {
return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1)
}
// parseErrorf returns a formatted parse error.
func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError {
return &ParseError{
Message: fmt.Sprintf(format, args...),
Pos: pos,
}
}
// decodeInt type converts v to target.
func decodeInt(v interface{}, target *int) error {
if v, ok := v.(uint64); ok {
*target = int(v)
return nil
}
return fmt.Errorf("invalid int value: %v", v)
}
// decodeUint64 type converts v to target.
func decodeUint64(v interface{}, target *uint64) error {
if v, ok := v.(uint64); ok {
*target = v
return nil
}
return fmt.Errorf("invalid int value: %v", v)
}
// decodeString type converts v to target.
func decodeString(v interface{}, target *string) error {
if v, ok := v.(string); ok {
*target = v
return nil
}
return fmt.Errorf("invalid string value: %v", v)
}
// decodeDate type converts v to target.
func decodeDate(v interface{}, target *time.Time) error {
if v, ok := v.(string); ok {
t, err := time.Parse(TimeFormat, v)
if err != nil {
return fmt.Errorf("invalid date format: %s", v)
}
*target = t
return nil
}
return fmt.Errorf("invalid date value: %v", v)
}

239
pql/parser_test.go Normal file
View file

@ -0,0 +1,239 @@
package pql_test
import (
"reflect"
"testing"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/pql"
)
// Ensure the parser can parse a "clear()" function with keyed args.
func TestParser_Parse_Clear_Key(t *testing.T) {
q, err := pql.ParseString(`clear(id=1, frame="b.n", filter=2, profile_id = 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Clear{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "clear()" function with array args.
func TestParser_Parse_Clear_Array(t *testing.T) {
q, err := pql.ParseString(`clear(1, "b.n", 2, 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Clear{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "count()" function.
func TestParser_Parse_Count(t *testing.T) {
q, err := pql.ParseString(`count(get(1))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Count{
Input: &pql.Get{
ID: 1,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "difference()" function.
func TestParser_Parse_Difference(t *testing.T) {
q, err := pql.ParseString(`difference(get(1), get(2))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Difference{
Inputs: pql.BitmapCalls{
&pql.Get{ID: 1},
&pql.Get{ID: 2},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "get()" function with keyed args.
func TestParser_Parse_Get_Key(t *testing.T) {
q, err := pql.ParseString(`get(id=1, frame="b.n")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Get{
ID: 1,
Frame: "b.n",
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "get()" function with array args.
func TestParser_Parse_Get_Array(t *testing.T) {
q, err := pql.ParseString(`get(1, "b.n")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Get{
ID: 1,
Frame: "b.n",
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "intersect()" function.
func TestParser_Parse_Intersect(t *testing.T) {
q, err := pql.ParseString(`intersect(get(1), get(2))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Intersect{
Inputs: pql.BitmapCalls{
&pql.Get{ID: 1},
&pql.Get{ID: 2},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "range()" function with keyed args.
func TestParser_Parse_Range_Key(t *testing.T) {
q, err := pql.ParseString(`range(start="2000-01-02T03:04", id=20, frame="b.n", end="2001-01-02T03:04")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "range()" function with array args.
func TestParser_Parse_Range_Array(t *testing.T) {
q, err := pql.ParseString(`range(20, "b.n", "2000-01-02T03:04", "2001-01-02T03:04")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "set()" function with keyed args.
func TestParser_Parse_Set_Key(t *testing.T) {
q, err := pql.ParseString(`set(id=1, frame="b.n", filter=2, profile_id = 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Set{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "set()" function with array args.
func TestParser_Parse_Set_Array(t *testing.T) {
q, err := pql.ParseString(`set(1, "b.n", 2, 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Set{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "top-n()" function with keyed args.
func TestParser_Parse_TopN_Key(t *testing.T) {
q, err := pql.ParseString(`top-n(frame="b.n", n=2)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Frame: "b.n",
N: 2,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "top-n()" function with array args.
func TestParser_Parse_TopN_Array(t *testing.T) {
q, err := pql.ParseString(`top-n("b.n", 2)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Frame: "b.n",
N: 2,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "union()" function.
func TestParser_Parse_Union(t *testing.T) {
q, err := pql.ParseString(`union(get(1), get(2))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Union{
Inputs: pql.BitmapCalls{
&pql.Get{ID: 1},
&pql.Get{ID: 2},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}

258
pql/scanner.go Normal file
View file

@ -0,0 +1,258 @@
package pql
import (
"bufio"
"bytes"
"io"
"unicode"
)
// Scanner represents a PQL lexical scanner.
type Scanner struct {
r io.RuneScanner
pos Pos
}
// NewScanner returns a new instance of Scanner.
func NewScanner(r io.Reader) *Scanner {
return &Scanner{r: bufio.NewReader(r)}
}
// Scan returns the next token and position from the underlying reader.
func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
pos = s.pos
// Read next code point.
ch := s.read()
// If we see whitespace then consume all contiguous whitespace.
// If we see a letter, or certain acceptable special characters, then consume
// as an ident or reserved word. If we see quotes, then scan as string.
if isWhitespace(ch) {
s.unread()
return s.scanWhitespace()
} else if isIdentFirstChar(ch) {
s.unread()
return s.scanIdent()
} else if isDigit(ch) {
s.unread()
return s.scanNumber()
} else if ch == '"' || ch == '\'' {
s.unread()
return s.scanString()
}
// Otherwise parse individual characters.
switch ch {
case eof:
tok = EOF
return
case '=':
tok = EQ
case ',':
tok = COMMA
case '(':
tok = LPAREN
case ')':
tok = RPAREN
case '[':
tok = LBRACK
case ']':
tok = RBRACK
default:
tok = ILLEGAL
}
lit = string(ch)
return
}
// read returns the next code point from the underlying reader and updates the pos.
func (s *Scanner) read() rune {
// Read next rune from underlying reader.
ch, _, err := s.r.ReadRune()
if err != nil {
return eof
}
// Update position information.
if ch == '\n' {
s.pos.Line++
s.pos.Char = 0
} else {
s.pos.Char++
}
return ch
}
// unread pushes the previously read rune back onto the reader.
func (s *Scanner) unread() {
if s.pos.Char == 0 {
s.pos.Line--
} else {
s.pos.Char--
}
s.r.UnreadRune()
}
// scanWhitespace consumes the current rune and all contiguous whitespace.
func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if ch == eof {
break
} else if !isWhitespace(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
return WS, pos, buf.String()
}
func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if ch == eof {
break
} else if !isIdentChar(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
lit = buf.String()
// If the literal matches a keyword then return that keyword.
if tok = Lookup(lit); tok != IDENT {
return tok, pos, lit
}
return IDENT, pos, lit
}
// scanNumber consumes consecutive integer digits.
func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if !isDigit(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
return NUMBER, pos, buf.String()
}
// scanString consumes a single-quoted or double-quoted string.
func (s *Scanner) scanString() (tok Token, pos Pos, lit string) {
pos = s.pos
// This must be either a single- or double-quote.
ending := s.read()
var buf bytes.Buffer
for {
ch := s.read()
if ch == ending {
break
} else if ch == '\n' || ch == eof {
return BADSTRING, pos, buf.String()
} else if ch == '\\' {
next := s.read()
if next == 'n' {
buf.WriteRune('\n')
} else if next == '\\' {
buf.WriteRune('\\')
} else if next == '"' {
buf.WriteRune('"')
} else if next == '\'' {
buf.WriteRune('\'')
} else {
return BADSTRING, pos, buf.String()
}
} else {
buf.WriteRune(ch)
}
}
return STRING, pos, buf.String()
}
// bufScanner represents a wrapper for scanner to add a buffer.
// It provides a fixed-length circular buffer that can be unread.
type bufScanner struct {
s *Scanner
i int // buffer index
n int // buffer size
buf [3]struct {
tok Token
pos Pos
lit string
}
}
// newBufScanner returns a new buffered scanner for a reader.
func newBufScanner(r io.Reader) *bufScanner {
return &bufScanner{s: NewScanner(r)}
}
// Scan reads the next token from the scanner.
func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) {
// If we have unread tokens then read them off the buffer first.
if s.n > 0 {
s.n--
return s.curr()
}
// Move buffer position forward and save the token.
s.i = (s.i + 1) % len(s.buf)
buf := &s.buf[s.i]
buf.tok, buf.pos, buf.lit = s.s.Scan()
return s.curr()
}
// unscan pushes the previously token back onto the buffer.
func (s *bufScanner) unscan() { s.n++ }
// curr returns the last read token.
func (s *bufScanner) curr() (tok Token, pos Pos, lit string) {
buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)]
return buf.tok, buf.pos, buf.lit
}
// pos returns the current position.
func (s *bufScanner) pos() Pos {
_, pos, _ := s.curr()
return pos
}
// isWhitespace returns true if the rune a Unicode space character.
func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) }
// isLetter returns true if the rune is a letter.
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
// isDigit returns true if the rune is a digit.
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
// isIdentChar returns true if the rune can be used in an unquoted identifier.
func isIdentChar(ch rune) bool { return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' }
// isIdentFirstChar returns true if the rune can be used as the first char in an identifier.
func isIdentFirstChar(ch rune) bool { return isLetter(ch) }
const eof = rune(0)

49
pql/scanner_test.go Normal file
View file

@ -0,0 +1,49 @@
package pql_test
import (
"strings"
"testing"
"github.com/umbel/pilosa/pql"
)
func TestScanner_Scan(t *testing.T) {
var tests = []struct {
s string
tok pql.Token
lit string
pos pql.Pos
}{
// Special tokens (EOF, ILLEGAL, WS)
{s: ``, tok: pql.EOF},
{s: `#`, tok: pql.ILLEGAL, lit: `#`},
{s: ` `, tok: pql.WS, lit: " "},
{s: "\t", tok: pql.WS, lit: "\t"},
{s: "\n", tok: pql.WS, lit: "\n"},
{s: `=`, tok: pql.EQ, lit: `=`},
{s: `,`, tok: pql.COMMA, lit: `,`},
{s: `(`, tok: pql.LPAREN, lit: `(`},
{s: `)`, tok: pql.RPAREN, lit: `)`},
{s: `[`, tok: pql.LBRACK, lit: `[`},
{s: `]`, tok: pql.RBRACK, lit: `]`},
{s: `foo`, tok: pql.IDENT, lit: `foo`},
{s: `100`, tok: pql.NUMBER, lit: `100`},
{s: `all`, tok: pql.ALL, lit: `all`},
{s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive
}
for i, tt := range tests {
s := pql.NewScanner(strings.NewReader(tt.s))
tok, pos, lit := s.Scan()
if tt.tok != tok {
t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit)
} else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char {
t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos)
} else if tt.lit != lit {
t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit)
}
}
}

81
pql/token.go Normal file
View file

@ -0,0 +1,81 @@
package pql
import "strings"
// Token is a lexical token of the PQL language.
type Token int
const (
// Special tokens
ILLEGAL Token = iota
EOF
WS
literal_beg
IDENT // main
STRING // "foo"
BADSTRING // bad escape or unclosed string
NUMBER // 12345
literal_end
keyword_beg
ALL
keyword_end
EQ // =
COMMA // ,
LPAREN // (
RPAREN // )
LBRACK // (
RBRACK // )
)
var tokens = [...]string{
ILLEGAL: "ILLEGAL",
EOF: "EOF",
WS: "WS",
IDENT: "IDENT",
NUMBER: "NUMBER",
ALL: "ALL",
EQ: "=",
COMMA: ",",
LPAREN: "(",
RPAREN: ")",
LBRACK: "(",
RBRACK: ")",
}
var keywords map[string]Token
func init() {
keywords = make(map[string]Token)
for tok := keyword_beg + 1; tok < keyword_end; tok++ {
keywords[strings.ToLower(tokens[tok])] = tok
}
}
// String returns the string representation of the token.
func (tok Token) String() string {
if tok >= 0 && tok < Token(len(tokens)) {
return tokens[tok]
}
return ""
}
// Lookup returns the token associated with a given string.
func Lookup(ident string) Token {
if tok, ok := keywords[strings.ToLower(ident)]; ok {
return tok
}
return IDENT
}
// Pos specifies the line and character position of a token.
// The Char and Line are both zero-based indexes.
type Pos struct {
Line int
Char int
}

348
query/lexer.go
View file

@ -1,348 +0,0 @@
package query
import (
"errors"
"fmt"
"strings"
"unicode"
"unicode/utf8"
log "github.com/cihub/seelog"
)
const (
TYPE_FUNC = iota
TYPE_LP = iota
TYPE_RP = iota
TYPE_LB = iota
TYPE_RB = iota
TYPE_VALUE = iota
TYPE_KEYWORD = iota
TYPE_EQUALS = iota
TYPE_COMMA = iota
TYPE_ERROR = iota
// Below types deprecated
TYPE_ID = iota
TYPE_FRAME = iota
TYPE_PROFILE = iota
TYPE_LIMIT = iota
)
type Token struct {
Text string
Type int
}
type statefn func(lexer *Lexer) statefn
type Lexer struct {
text string // the string being scanned.
pos int // current position in the input.
width int // width of last rune read from input.
start int // start position of this item.
state int // current state of lexer NEEDED???
ch chan Token // channel of scanned items (Tokens).
}
func (lexer *Lexer) emit(typ int) {
log.Trace("Lexer.emit", typ)
if lexer.start < lexer.pos {
lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ}
}
lexer.start = lexer.pos
}
func (lexer *Lexer) acceptUntil(chars string, consume bool) (rune, error) {
log.Trace("Lexer.acceptUntil", chars, consume)
start_pos := lexer.pos
for {
next := lexer.next()
if next == rune(' ') {
lexer.ignore()
}
if next == 0 {
lexer.pos = start_pos
return 0, errors.New("Not found")
}
ch := strings.IndexRune(chars, next)
if ch >= 0 {
if consume {
lexer.backup()
} else {
lexer.pos = start_pos
}
return []rune(chars)[ch], nil
}
}
}
func (lexer *Lexer) acceptRun(valid string) {
log.Trace("Lexer.acceptRun", valid)
for strings.IndexRune(valid, lexer.next()) >= 0 {
}
lexer.backup()
}
// next returns the next rune in the input.
func (lexer *Lexer) next() (runey rune) {
log.Trace("Lexer.next", runey)
if lexer.pos >= len(lexer.text) {
lexer.width = 0
return 0
}
runey, lexer.width = utf8.DecodeRuneInString(lexer.text[lexer.pos:])
lexer.pos += lexer.width
return runey
}
// ignore skips over the pending input before this point.
func (lexer *Lexer) ignore() {
log.Trace("Lexer.ignore")
lexer.start = lexer.pos
}
// backup steps back one rune.
// Can be called only once per call of next.
func (lexer *Lexer) backup() {
log.Trace("Lexer.backup")
lexer.pos -= lexer.width
}
// peek returns but does not consume
// the next rune in the input.
func (lexer *Lexer) peek() rune {
log.Trace("Lexer.peek")
for {
next_rune := lexer.next()
// ignore spaces
if next_rune != rune(' ') {
lexer.backup()
return next_rune
}
lexer.ignore()
}
}
func stateError(err error) func(lexer *Lexer) statefn {
log.Trace("stateError", err)
return func(lexer *Lexer) statefn {
lexer.ch <- Token{err.Error(), TYPE_ERROR}
close(lexer.ch)
return nil
}
}
func stateFunc(lexer *Lexer) statefn {
log.Trace("stateFunc", lexer)
_, err := lexer.acceptUntil("(", true)
if err != nil {
return stateError(err)
}
lexer.emit(TYPE_FUNC)
return stateLP
}
func stateLP(lexer *Lexer) statefn {
log.Trace("stateLP", lexer)
lexer.pos += 1
lexer.emit(TYPE_LP)
// handle multiple LPs
if lexer.peek() == rune('(') {
return stateLP
}
return stateArgs
}
func stateLB(lexer *Lexer) statefn {
log.Trace("stateLB", lexer)
lexer.acceptUntil("[", true)
lexer.next()
lexer.emit(TYPE_LB)
peeked := lexer.peek()
if peeked == rune(']') {
lexer.next()
lexer.emit(TYPE_RB)
return stateArgs
} else if unicode.IsDigit(peeked) {
for {
r, err := lexer.acceptUntil(",]", true)
if err != nil {
return stateError(errors.New("Unclosed bracket!"))
}
lexer.emit(TYPE_VALUE)
if r == ',' {
lexer.next()
lexer.emit(TYPE_COMMA)
} else {
lexer.next()
lexer.emit(TYPE_RB)
return stateArgs
}
}
} else {
return stateArgs
}
}
func stateRB(lexer *Lexer) statefn {
log.Trace("stateRB", lexer)
lexer.pos += 1
lexer.emit(TYPE_RB)
peeked := lexer.peek()
if peeked == rune(',') {
return stateRPComma
} else if peeked == rune(')') {
return stateRP
} else {
return stateEOF
}
}
func stateArgs(lexer *Lexer) statefn {
log.Trace("stateArgs", lexer)
r, err := lexer.acceptUntil("(),=[]", false)
if err != nil {
return stateError(err)
}
switch r {
case '(':
return stateFunc
case ')':
return stateValue
case ',':
return stateValue
case '=':
return stateKeyword
case '[':
return stateLB
case ']':
return stateRB
default:
return stateError(errors.New("Expecting arguments!"))
}
}
func stateKeyword(lexer *Lexer) statefn {
log.Trace("stateKeyword", lexer)
_, err := lexer.acceptUntil("=", true)
if err != nil {
return stateError(err)
}
lexer.emit(TYPE_KEYWORD)
return stateEquals
}
func stateEquals(lexer *Lexer) statefn {
log.Trace("stateEquals", lexer)
e := lexer.next()
if e != '=' {
return stateError(errors.New("Expecting '='!"))
}
lexer.emit(TYPE_EQUALS)
return stateValue
}
func stateValue(lexer *Lexer) statefn {
log.Trace("stateValue", lexer)
r, err := lexer.acceptUntil("(),[", false)
if err != nil {
return stateError(err)
}
switch r {
case '(':
return stateFunc
case ')':
lexer.acceptUntil(")", true)
if lexer.pos > lexer.start {
lexer.emit(TYPE_VALUE)
}
return stateRP
case ',':
lexer.acceptUntil(",", true)
lexer.emit(TYPE_VALUE)
return stateComma
case '[':
return stateLB
default:
return stateError(errors.New("Unexpected character!"))
}
}
func stateRP(lexer *Lexer) statefn {
log.Trace("stateRP", lexer)
lexer.pos += 1
lexer.emit(TYPE_RP)
peeked := lexer.peek()
if peeked == rune(',') {
return stateRPComma
} else if peeked == rune(')') {
return stateRP
} else if peeked == rune(']') {
return stateRB
} else {
return stateEOF
}
}
func stateRPComma(lexer *Lexer) statefn {
log.Trace("stateRPComma", lexer)
lexer.pos += 1
lexer.emit(TYPE_COMMA)
return stateValue
}
func stateComma(lexer *Lexer) statefn {
log.Trace("stateComma", lexer)
lexer.pos += 1
lexer.emit(TYPE_COMMA)
return stateArgs
}
func stateEOF(lexer *Lexer) statefn {
log.Trace("stateEOF", lexer)
close(lexer.ch)
return nil
}
func (lexer *Lexer) Lex() (tokens []Token, err error) {
log.Trace("Lexer.Lex", tokens, err)
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("query: %v", r)
}
}
}()
tokens = make([]Token, 0)
state := stateFunc
go func() {
for {
state = state(lexer)
if state == nil {
return
}
}
}()
for t := range lexer.ch {
if t.Type == TYPE_ERROR {
err = errors.New(t.Text)
}
tokens = append(tokens, t)
}
return tokens, err
}
func Lex(input string) ([]Token, error) {
log.Trace("Lex", input)
lexer := Lexer{input, 0, 0, 0, TYPE_FUNC, make(chan Token)}
return lexer.Lex()
}

279
query/lexer_test.go
View file

@ -1,279 +0,0 @@
package query_test
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/query"
)
// Ensure a simple function and value can be lexed.
func TestLexer_Lex_FuncValue(t *testing.T) {
if tokens, err := query.Lex("get(10)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure a simple function, keyword, & value can be lexed.
func TestLexer_Lex_FuncKeywordValue(t *testing.T) {
if tokens, err := query.Lex("get(id=10)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"id", query.TYPE_KEYWORD},
{"=", query.TYPE_EQUALS},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure a more complex function can be lexed.
func TestLexer_Lex_FuncComplex(t *testing.T) {
if tokens, err := query.Lex("get(id=10, frame=brand)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"id", query.TYPE_KEYWORD},
{"=", query.TYPE_EQUALS},
{"10", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"frame", query.TYPE_KEYWORD},
{"=", query.TYPE_EQUALS},
{"brand", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that nested functions can be lexed.
func TestLexer_Lex_FuncNested(t *testing.T) {
if tokens, err := query.Lex("union(get(10))"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"union", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a list of nested functions can be lexed.
func TestLexer_Lex_FuncNestedList(t *testing.T) {
if tokens, err := query.Lex("intersect(get(10), get(11), get(12))"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"intersect", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"11", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"12", query.TYPE_VALUE},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that complex nested functions can be lexed.
func TestLexer_Lex_FuncNestedComplex(t *testing.T) {
if tokens, err := query.Lex("intersect(get(10), get(11), concat(get(12),get(14)))"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"intersect", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"11", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"concat", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"12", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"14", query.TYPE_VALUE},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that complex nested functions can be lexed.
func TestLexer_Lex_FuncNestedComplex2(t *testing.T) {
if tokens, err := query.Lex("concat(get(1, brand),get(2))"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"concat", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"1", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"brand", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"2", query.TYPE_VALUE},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a set function can be lexed.
func TestLexer_Lex_Set1(t *testing.T) {
if tokens, err := query.Lex("set(1, 987)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"set", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"1", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"987", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a set function can be lexed.
func TestLexer_Lex_Set2(t *testing.T) {
if tokens, err := query.Lex("set(1, general, 987)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"set", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"1", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"general", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"987", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a TopN function can be lexed.
func TestLexer_Lex_TopN1(t *testing.T) {
if tokens, err := query.Lex("top-n(get(10), 8)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"top-n", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"8", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a TopN function can be lexed.
func TestLexer_Lex_TopN2(t *testing.T) {
if tokens, err := query.Lex("top-n(get(10, general), [1,2,3])"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"top-n", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"general", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"[", query.TYPE_LB},
{"1", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"2", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"3", query.TYPE_VALUE},
{"]", query.TYPE_RB},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a plugin function can be lexed.
func TestLexer_Lex_Plugin(t *testing.T) {
if tokens, err := query.Lex("plugin(get(10, general), [get(11, general)])"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"plugin", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"general", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"[", query.TYPE_LB},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"11", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"general", query.TYPE_VALUE},
{")", query.TYPE_RP},
{"]", query.TYPE_RB},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// MustLex lexes s and returns a set of tokens. Panic on error.
func MustLex(s string) []query.Token {
a, err := query.Lex(s)
if err != nil {
panic(err)
}
return a
}

354
query/parser.go
View file

@ -1,354 +0,0 @@
package query
import (
"errors"
"fmt"
"strconv"
"time"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
)
var InvalidQueryError = errors.New("Invalid query format.")
type QueryParser struct {
tokens []Token
pos int
}
func (self *QueryParser) next() *Token {
log.Trace("QueryParser.next")
self.pos += 1
if self.pos > len(self.tokens) {
return nil
}
return &self.tokens[self.pos-1]
}
func (self *QueryParser) peek() *Token {
log.Trace("QueryParser.peek")
token := self.next()
self.backup()
return token
}
func (self *QueryParser) backup() {
log.Trace("QueryParser.backup")
self.pos -= 1
}
func (self *QueryParser) Parse() (query *Query, err error) {
log.Trace("QueryParser.Parse", query, err)
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("query: %v", r)
}
}
}()
var token *Token
id := pilosa.NewGUID()
query = &Query{Id: &id, Subqueries: make([]Query, 0), Args: make(map[string]interface{})}
token = self.next()
if token.Type != TYPE_FUNC {
return nil, fmt.Errorf("Expected function, found token %v.", token)
}
query.Operation = token.Text
token = self.next()
if token.Type != TYPE_LP {
return nil, fmt.Errorf("Expected '(', found token %v.", token)
}
const shortForm = "2006-01-02T15:04"
ArgLoop:
for {
token = self.next()
if token == nil {
return nil, fmt.Errorf("Unclosed parentheses!")
}
switch token.Type {
case TYPE_FUNC:
self.backup()
subquery, err := self.Parse()
if err != nil {
return nil, err
}
query.Subqueries = append(query.Subqueries, *subquery)
case TYPE_VALUE:
switch query.Operation {
case "range":
switch len(query.Args) {
case 0:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["id"] = i
case 1:
query.Args["frame"] = token.Text
case 2:
t, err := time.Parse(shortForm, token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer DateTime (%v)", err)
}
query.Args["start"] = t
case 3:
t, err := time.Parse(shortForm, token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer DateTime (%v)", err)
}
query.Args["end"] = t
}
case "mask":
switch len(query.Args) {
case 0:
query.Args["frame"] = token.Text
case 1:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["start"] = i
case 2:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["end"] = i
}
case "get":
switch len(query.Args) {
case 0:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["id"] = i
case 1:
query.Args["frame"] = token.Text
default:
return nil, fmt.Errorf("Unexpected argument! (%v)", token)
}
case "clear":
switch len(query.Args) {
case 0:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["id"] = i
case 1:
query.Args["frame"] = token.Text
case 2:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["filter"] = i
case 3:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["profile_id"] = i
default:
return nil, fmt.Errorf("Unexpected argument! (%v)", token)
}
case "set":
switch len(query.Args) {
case 0:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["id"] = i
case 1:
query.Args["frame"] = token.Text
case 2:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["filter"] = i
case 3:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["profile_id"] = i
default:
return nil, fmt.Errorf("Unexpected argument! (%v)", token)
}
case "top-n":
switch len(query.Args) {
case 0:
query.Args["frame"] = token.Text
case 1:
i, err := strconv.Atoi(token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer! (%v)", err)
}
query.Args["n"] = i
}
case "all":
// do nothing
case "recall": //need pair based list of frag_id,handle
arg, ok := query.Args["stash"]
if !ok {
arg = &Stash{make([]CacheItem, 0), false}
query.Args["stash"] = arg
}
recallArgs := arg.(*Stash)
if !recallArgs.incomplete {
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting fragment id! (%v)", err)
}
recallArgs.Add(pilosa.SUUID(i)) //constructs a new arg
} else {
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting handle! (%v)", err)
}
recallArgs.Assign(pilosa.BitmapHandle(i)) //sets the value of the last created arg
}
default:
spew.Dump("UNPROCESSED VALUE", token)
}
continue
case TYPE_COMMA:
continue
case TYPE_RP:
break ArgLoop
case TYPE_KEYWORD:
var value interface{}
keyword := token.Text
token = self.next()
if token == nil || token.Type != TYPE_EQUALS {
return nil, fmt.Errorf("Expecting equals sign!")
}
token = self.next()
if token == nil || token.Type != TYPE_VALUE {
return nil, fmt.Errorf("Expecting value!")
}
if keyword == "id" {
value, err = strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
} else if keyword == "n" {
value, err = strconv.Atoi(token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer! (%v)", err)
}
} else {
value = token.Text
}
query.Args[keyword] = value
case TYPE_LB:
peeked := self.peek()
// we currently support 2 types of values in square brackets:
// ids <- list of integers (TYPE_VALUE)
// filters <- list of queries (TYPE_FUNC)
switch peeked.Type {
case TYPE_VALUE:
query.Args["ids"] = make([]uint64, 0)
for {
token = self.next()
if token == nil {
return nil, fmt.Errorf("Unclosed list!")
}
switch token.Type {
case TYPE_COMMA:
break
case TYPE_VALUE:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["ids"] = append(query.Args["ids"].([]uint64), i)
case TYPE_RB:
continue ArgLoop
default:
return nil, fmt.Errorf("Unexpected token! (%v)", token)
}
}
case TYPE_FUNC:
query.Args["filters"] = make([]Query, 0)
for {
token = self.next()
if token == nil {
return nil, fmt.Errorf("Unclosed list!")
}
switch token.Type {
case TYPE_COMMA:
break
case TYPE_FUNC:
self.backup()
filterquery, err := self.Parse()
if err != nil {
return nil, err
}
query.Args["filters"] = append(query.Args["filters"].([]Query), *filterquery)
case TYPE_RB:
continue ArgLoop
default:
return nil, fmt.Errorf("Unexpected token! (%v)", token)
}
}
}
case TYPE_RB:
//
default:
log.Warn(spew.Sdump("unexpected", token))
return nil, errors.New("BAD TOKEN")
}
}
if query.Operation == "get" && query.Args["frame"] == nil {
query.Args["frame"] = "general"
}
if len(query.Args) == 0 && len(query.Subqueries) == 0 {
if query.Operation == "count" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "intersect" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "union" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "difference" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "range" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "mask" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "stash" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "recall" {
return nil, fmt.Errorf("No Args Given")
}
}
return query, nil
}
func Parse(tokens []Token) (*Query, error) {
log.Trace("Parse", tokens)
parser := QueryParser{tokens, 0}
return parser.Parse()
}

157
query/parser_test.go
View file

@ -1,157 +0,0 @@
package query_test
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/query"
)
// Ensure the parser can parse a get() query.
func TestParser_Parse_Get(t *testing.T) {
if q, err := query.Parse(MustLex("get(10)")); err != nil {
t.Fatal(err)
} else if q.Operation != "get" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{
"id": uint64(10),
"frame": "general",
}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a get() query with a keyword.
func TestParser_Parse_Get_Keyword(t *testing.T) {
if q, err := query.Parse(MustLex("get(id=10)")); err != nil {
t.Fatal(err)
} else if q.Operation != "get" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{"id": uint64(10), "frame": "general"}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a get() query with a keyword.
func TestParser_Parse_Get_MultiKeyword(t *testing.T) {
if q, err := query.Parse(MustLex("get(id=10, frame=brands)")); err != nil {
t.Fatal(err)
} else if q.Operation != "get" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{"id": uint64(10), "frame": "brands"}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a clear() query.
func TestParser_Parse_Clear(t *testing.T) {
if q, err := query.Parse(MustLex("clear(10, general, 0, 20)")); err != nil {
t.Fatal(err)
} else if q.Operation != "clear" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{
"id": uint64(10),
"frame": "general",
"filter": uint64(0),
"profile_id": uint64(20),
}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a set() query.
func TestParser_Parse_Set(t *testing.T) {
if q, err := query.Parse(MustLex("set(10, general, 0, 20)")); err != nil {
t.Fatal(err)
} else if q.Operation != "set" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{
"id": uint64(10),
"frame": "general",
"filter": uint64(0),
"profile_id": uint64(20),
}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a nested query.
func TestParser_Parse_Nested(t *testing.T) {
q, err := query.Parse(MustLex("union(get(10,general), get(11,brand), get(12))"))
if err != nil {
t.Fatal(err)
} else if q.Operation != "union" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if len(q.Subqueries) != 3 {
t.Fatalf("unexpected subquery count: %d", len(q.Subqueries))
}
if sq := q.Subqueries[0]; sq.Operation != "get" {
t.Fatalf("unexpected subquery(0) operation: %q", sq.Operation)
} else if !reflect.DeepEqual(sq.Args, map[string]interface{}{"id": uint64(10), "frame": "general"}) {
t.Fatalf("unexpected subquery(0) args:\n\n%s", spew.Sprint(q.Args))
}
if sq := q.Subqueries[1]; sq.Operation != "get" {
t.Fatalf("unexpected subquery(1) operation: %q", sq.Operation)
} else if !reflect.DeepEqual(sq.Args, map[string]interface{}{"id": uint64(11), "frame": "brand"}) {
t.Fatalf("unexpected subquery(1) args:\n\n%s", spew.Sprint(sq.Args))
}
if sq := q.Subqueries[2]; sq.Operation != "get" {
t.Fatalf("unexpected subquery(2) operation: %q", sq.Operation)
} else if !reflect.DeepEqual(sq.Args, map[string]interface{}{"id": uint64(12), "frame": "general"}) {
t.Fatalf("unexpected subquery(2) args:\n\n%s", spew.Sprint(sq.Args))
}
}
// Ensure the parser can parse a query with lists.
func TestParser_Parse_Lists(t *testing.T) {
q, err := query.Parse(MustLex("top-n(get(10, general), [1,2,3], 50)"))
if err != nil {
t.Fatal(err)
} else if q.Operation != "top-n" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{"ids": []uint64{1, 2, 3}, "n": 50}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
if sq := q.Subqueries[0]; sq.Operation != "get" {
t.Fatalf("unexpected subquery(0) operation: %q", sq.Operation)
} else if !reflect.DeepEqual(sq.Args, map[string]interface{}{"id": uint64(10), "frame": "general"}) {
t.Fatalf("unexpected subquery(0) args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse "all()".
func TestParser_Parse_All(t *testing.T) {
q, err := query.Parse(MustLex("top-n(all(), general, 30)"))
if err != nil {
t.Fatal(err)
} else if q.Operation != "top-n" {
t.Fatalf("unexpected operation: %q", q.Operation)
}
if sq := q.Subqueries[0]; sq.Operation != "all" {
t.Fatalf("unexpected subquery(0) operation: %q", sq.Operation)
}
}
// Ensure the parser can parse bracketed lists.
func TestParser_Parse_Lists_Bracketed(t *testing.T) {
q, err := query.Parse(MustLex("plugin(get(99), [get(10), get(11)])"))
if err != nil {
t.Fatalf("expected error")
}
spew.Dump(q)
}
// Ensure the parser can parse a recall query.
func TestParser_Parse_Recall(t *testing.T) {
q, err := query.Parse(MustLex("recall(12345,1,12345,2,12345,3)"))
if err != nil {
t.Fatal(err)
}
spew.Dump(q)
}

953
query/planner.go
View file

@ -1,953 +0,0 @@
package query
import (
"encoding/gob"
"errors"
"fmt"
"math/rand"
"time"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
type PortableQueryStep interface {
GetId() *pilosa.GUID
GetLocation() *db.Location
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// ERRORS
///////////////////////////////////////////////////////////////////////////////////////////////////
// Invalid Frame
type InvalidFrame struct {
Db string
Frame string
Retry bool
}
func NewInvalidFrame(db string, frame string) *InvalidFrame {
return &InvalidFrame{db, frame, false}
}
func (self *InvalidFrame) Error() string {
return fmt.Sprintf("Invalid Frame: %s:%s", self.Db, self.Frame)
}
// Fragment Not Found
type FragmentNotFound struct {
Db string
Frame string
Slice int
Retry bool
}
func NewFragmentNotFound(db string, frame string, slice int) *FragmentNotFound {
return &FragmentNotFound{db, frame, slice, true}
}
func (self *FragmentNotFound) Error() string {
return fmt.Sprintf("Fragment Not Found: %s:%s:%d", self.Db, self.Frame, self.Slice)
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// BASE
///////////////////////////////////////////////////////////////////////////////////////////////////
type BaseQueryStep struct {
Id *pilosa.GUID
Operation string
Location *db.Location
Destination *db.Location
}
func (self *BaseQueryStep) GetId() *pilosa.GUID {
log.Trace("BaseQueryStep", *self.Id)
return self.Id
}
func (self *BaseQueryStep) GetLocation() *db.Location {
log.Trace("BaseQueryStep.GetLocation", *self.Location)
return self.Location
}
func (self *BaseQueryStep) LocIsDest() bool {
log.Trace("BaseQueryStep.LocIsDest")
if self.Location.ProcessId.Equals(self.Destination.ProcessId) &&
self.Location.FragmentId == self.Destination.FragmentId {
log.Trace("BaseQueryStep.LocIsDest Return true")
return true
}
log.Trace("BaseQueryStep.LocIsDest Return false")
return false
}
type BaseQueryResult struct {
Id *pilosa.GUID
Data interface{}
}
func (self *BaseQueryResult) ResultId() *pilosa.GUID {
log.Trace("BaseQueryStep.ResultId", self)
return self.Id
}
func (self *BaseQueryResult) ResultData() interface{} {
log.Trace("BaseQueryStep.ResultData", self)
return self.Data
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// COUNT
///////////////////////////////////////////////////////////////////////////////////////////////////
type CountQueryStep struct {
*BaseQueryStep
Input *pilosa.GUID
}
type CountQueryResult struct {
*BaseQueryResult
}
// QueryTree for COUNT queries
type CountQueryTree struct {
subquery QueryTree
location *db.Location
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *CountQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("CountQueryTree.getLocation", d, qt)
return qt.subquery.getLocation(d)
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// TOP-N
///////////////////////////////////////////////////////////////////////////////////////////////////
type TopNQueryStep struct {
*BaseQueryStep
Input *pilosa.GUID
Filters []uint64
N int
Frame string
}
type TopNQueryResult struct {
*BaseQueryResult
}
// QueryTree for TOP-N queries
type TopNQueryTree struct {
subquery QueryTree
location *db.Location
Filters []uint64
N int
Frame string
Slice int
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *TopNQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("TopNQueryTree.getLocation", d, qt)
var err error
if qt.location == nil {
frame := d.GetOrCreateFrame(qt.Frame)
slice := d.GetOrCreateSlice(qt.Slice)
fragment, err := d.GetFragmentForFrameSlice(frame, slice)
if err != nil {
log.Warn("GetFragmentForFrameSliceFailed TopNQueryTree", frame, slice)
return nil, err
}
qt.location = fragment.GetLocation()
return qt.location, nil
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// UNION
///////////////////////////////////////////////////////////////////////////////////////////////////
type UnionQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
}
type UnionQueryResult struct {
*BaseQueryResult
}
// QueryTree for UNION queries
type UnionQueryTree struct {
subqueries []QueryTree
location *db.Location
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *UnionQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("UnionQueryTree.getLocation", d, qt)
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// INTERSECT
///////////////////////////////////////////////////////////////////////////////////////////////////
type IntersectQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
}
type IntersectQueryResult struct {
*BaseQueryResult
}
// QueryTree for UNION queries
type IntersectQueryTree struct {
subqueries []QueryTree
location *db.Location
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *IntersectQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("IntersectQueryTree.getLocation", d, qt)
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// DIFFERENCE
///////////////////////////////////////////////////////////////////////////////////////////////////
type DifferenceQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
}
type DifferenceQueryResult struct {
*BaseQueryResult
}
// QueryTree for UNION queries
type DifferenceQueryTree struct {
subqueries []QueryTree
location *db.Location
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *DifferenceQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("DifferenceQueryTree.getLocation", d, qt)
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// CAT
///////////////////////////////////////////////////////////////////////////////////////////////////
type CatQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
N int
}
type Appendable interface {
Append(subq QueryTree)
}
type CatQueryResult struct {
*BaseQueryResult
}
// QueryTree for CAT queries
type CatQueryTree struct {
subqueries []QueryTree
location *db.Location
N int
}
func (qt *CatQueryTree) Append(subtree QueryTree) {
log.Trace("CatQueryTree.Append", qt, subtree)
qt.subqueries = append(qt.subqueries, subtree)
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *CatQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("CatQueryTree.getLocation", d)
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// ALL
///////////////////////////////////////////////////////////////////////////////////////////////////
// QueryTree for GET queries
type AllQueryTree struct {
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *AllQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("AllQueryTree.getLocation", d)
return nil, nil
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// GET
///////////////////////////////////////////////////////////////////////////////////////////////////
type GetQueryStep struct {
*BaseQueryStep
Bitmap *db.Bitmap
Slice int
}
type GetQueryResult struct {
*BaseQueryResult
}
// QueryTree for GET queries
type GetQueryTree struct {
bitmap *db.Bitmap
slice int
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *GetQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("GetQueryTree.getLocation", d)
slice := d.GetOrCreateSlice(qt.slice) // TODO: this should probably be just GetSlice (no create)
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("GetFragmenForBitmapFailed GetQueryTree", slice)
return nil, err
}
return fragment.GetLocation(), nil
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// SET
///////////////////////////////////////////////////////////////////////////////////////////////////
type SetQueryStep struct {
*BaseQueryStep
Bitmap *db.Bitmap
ProfileId uint64
}
type SetQueryResult struct {
*BaseQueryResult
}
// QueryTree for SET queries
type SetQueryTree struct {
bitmap *db.Bitmap
profile_id uint64
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *SetQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("SetQueryTree.getLocation", d)
// check here for supported frames
if !d.IsValidFrame(qt.bitmap.FrameType) {
return nil, NewInvalidFrame(d.Name, qt.bitmap.FrameType)
}
slice, err := d.GetSliceForProfile(qt.profile_id)
if err != nil {
return nil, NewFragmentNotFound(d.Name, qt.bitmap.FrameType, db.GetSlice(qt.profile_id))
}
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("NOT FOUND:", slice, qt.bitmap)
return nil, NewFragmentNotFound(d.Name, qt.bitmap.FrameType, db.GetSlice(qt.profile_id))
}
return fragment.GetLocation(), nil
}
///////////////////////////////////////////////////////////////////////////////////////////////////
func init() {
gob.Register(BaseQueryResult{})
gob.Register(SetQueryResult{})
gob.Register(ClearQueryResult{})
gob.Register(GetQueryResult{})
gob.Register(RangeQueryResult{})
gob.Register(CatQueryResult{})
gob.Register(UnionQueryResult{})
gob.Register(IntersectQueryResult{})
gob.Register(DifferenceQueryResult{})
gob.Register(CountQueryResult{})
gob.Register(TopNQueryResult{})
gob.Register(FillResult{})
gob.Register(StashQueryResult{})
gob.Register(CacheItem{})
gob.Register(Stash{})
gob.Register(SetQueryStep{})
gob.Register(ClearQueryStep{})
gob.Register(GetQueryStep{})
gob.Register(RangeQueryStep{})
gob.Register(CatQueryStep{})
gob.Register(UnionQueryStep{})
gob.Register(IntersectQueryStep{})
gob.Register(DifferenceQueryStep{})
gob.Register(CountQueryStep{})
gob.Register(TopNQueryStep{})
gob.Register(StashQueryStep{})
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// This is the output of the query planner. Contains a list of steps which can be performed in parallel
//type QueryPlan []QueryStep
type QueryPlan []interface{}
type QueryPlanner struct {
Database *db.Database
Query *Query
}
type QueryTree interface {
getLocation(d *db.Database) (*db.Location, error)
}
func validateRange(Args map[string]interface{}) error {
log.Trace("validateRange", Args)
_, ok := Args["id"]
if !ok {
return errors.New("missing bitmap id")
}
_, ok = Args["frame"]
if !ok {
return errors.New("missing frame ")
}
_, ok = Args["start"]
if !ok {
return errors.New("missing start time")
}
_, ok = Args["end"]
if !ok {
return errors.New("missing end time")
}
return nil
}
// Builds QueryTree object from Query. Pass slice=-1 to perform operation on all slices
func (self *QueryPlanner) buildTree(query *Query, slice int) (QueryTree, error) {
log.Trace("QueryPlanner.buildTree", query, slice)
var tree QueryTree
// handle SET operation regardless of the slice
if query.Operation == "set" {
tree = &SetQueryTree{&db.Bitmap{Id: query.Args["id"].(uint64), FrameType: query.Args["frame"].(string), Filter: query.Args["filter"].(uint64)}, query.Args["profile_id"].(uint64)}
return tree, nil
}
if query.Operation == "clear" {
tree = &ClearQueryTree{&db.Bitmap{Id: query.Args["id"].(uint64), FrameType: query.Args["frame"].(string), Filter: query.Args["filter"].(uint64)}, query.Args["profile_id"].(uint64)}
return tree, nil
}
if query.Operation == "recall" {
tree = &RecallQueryTree{query.Args["stash"].(Stash)}
return tree, nil
}
// handle the remaining operations, taking slice into consideration
//I'm kinda thinking for stash it needs a differnt handler..i guess for now I'll see if i can use the cathandler
if slice == -1 {
var n int
n_, ok := query.Args["n"]
if ok {
n = n_.(int)
}
var p Appendable
if query.Operation == "stash" {
tree = &StashQueryTree{N: n}
} else {
tree = &CatQueryTree{N: n}
}
p = tree.(Appendable)
slice_ids, err := self.Database.SliceIds()
if err != nil {
return nil, err
}
for i := range slice_ids {
subtree, err := self.buildTree(query, slice_ids[i])
if err != nil {
return nil, err
}
p.Append(subtree)
}
} else {
if query.Operation == "get" {
tree = &GetQueryTree{&db.Bitmap{Id: query.Args["id"].(uint64), FrameType: query.Args["frame"].(string), Filter: 0}, slice}
return tree, nil
} else if query.Operation == "range" {
err := validateRange(query.Args)
if err != nil {
return nil, err
}
tree = &RangeQueryTree{
&db.Bitmap{
Id: query.Args["id"].(uint64),
FrameType: query.Args["frame"].(string),
Filter: 0},
slice,
query.Args["start"].(time.Time),
query.Args["end"].(time.Time)}
return tree, nil
} else if query.Operation == "count" {
subquery, err := self.buildTree(&query.Subqueries[0], slice)
if err != nil {
return nil, err
}
tree = &CountQueryTree{subquery: subquery}
} else if query.Operation == "all" {
tree = &AllQueryTree{}
} else if query.Operation == "top-n" {
var frame string
var n int
var filters []uint64
frame_, ok := query.Args["frame"]
if ok {
frame = frame_.(string)
}
n_, ok := query.Args["n"]
if ok {
n = n_.(int)
}
filters_, ok := query.Args["ids"]
if ok {
filters = filters_.([]uint64)
}
subquery, err := self.buildTree(&query.Subqueries[0], slice)
if err != nil {
return nil, err
}
tree = &TopNQueryTree{subquery: subquery, Filters: filters, N: n, Frame: frame, Slice: slice}
} else if query.Operation == "union" {
subqueries := make([]QueryTree, len(query.Subqueries))
var err error
for i, query := range query.Subqueries {
subqueries[i], err = self.buildTree(&query, slice)
if err != nil {
return nil, err
}
}
tree = &UnionQueryTree{subqueries: subqueries}
} else if query.Operation == "intersect" {
subqueries := make([]QueryTree, len(query.Subqueries))
var err error
for i, query := range query.Subqueries {
subqueries[i], err = self.buildTree(&query, slice)
if err != nil {
return nil, err
}
}
tree = &IntersectQueryTree{subqueries: subqueries}
} else if query.Operation == "difference" {
subqueries := make([]QueryTree, len(query.Subqueries))
var err error
for i, query := range query.Subqueries {
subqueries[i], err = self.buildTree(&query, slice)
if err != nil {
return nil, err
}
}
tree = &DifferenceQueryTree{subqueries: subqueries}
} else if query.Operation == "stash" {
subqueries := make([]QueryTree, len(query.Subqueries))
var err error
for i, query := range query.Subqueries {
subqueries[i], err = self.buildTree(&query, slice)
if err != nil {
return nil, err
}
}
tree = &StashQueryTree{subqueries: subqueries}
} else {
//TODO return error gracefully
log.Warn(spew.Sdump(query))
return nil, errors.New("BuildTree Issues")
}
}
return tree, nil
}
// Produces flattened QueryPlan from QueryTree input
func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanner.flatten", self, qt, id, location)
plan := QueryPlan{}
if cat, ok := qt.(*CatQueryTree); ok {
inputs := make([]*pilosa.GUID, len(cat.subqueries))
loc, err := cat.getLocation(self.Database)
if err != nil {
return nil, err
}
step := CatQueryStep{&BaseQueryStep{id, "cat", loc, location}, inputs, cat.N}
for index, subq := range cat.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if stash, ok := qt.(*StashQueryTree); ok {
inputs := make([]*pilosa.GUID, len(stash.subqueries))
loc, err := stash.getLocation(self.Database)
if err != nil {
return nil, err
}
step := StashQueryStep{&BaseQueryStep{id, "stash", loc, location}, inputs, stash.N}
for index, subq := range stash.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if union, ok := qt.(*UnionQueryTree); ok {
inputs := make([]*pilosa.GUID, len(union.subqueries))
loc, err := union.getLocation(self.Database)
if err != nil {
return nil, err
}
step := UnionQueryStep{&BaseQueryStep{id, "union", loc, location}, inputs}
for index, subq := range union.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if intersect, ok := qt.(*IntersectQueryTree); ok {
inputs := make([]*pilosa.GUID, len(intersect.subqueries))
loc, err := intersect.getLocation(self.Database)
if err != nil {
return nil, err
}
step := IntersectQueryStep{&BaseQueryStep{id, "intersect", loc, location}, inputs}
for index, subq := range intersect.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if difference, ok := qt.(*DifferenceQueryTree); ok {
inputs := make([]*pilosa.GUID, len(difference.subqueries))
loc, err := difference.getLocation(self.Database)
if err != nil {
return nil, err
}
step := DifferenceQueryStep{&BaseQueryStep{id, "difference", loc, location}, inputs}
for index, subq := range difference.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if get, ok := qt.(*GetQueryTree); ok {
loc, err := get.getLocation(self.Database)
if err != nil {
return nil, err
}
step := GetQueryStep{&BaseQueryStep{id, "get", loc, location}, get.bitmap, get.slice}
plan := QueryPlan{step}
return &plan, nil
} else if rang, ok := qt.(*RangeQueryTree); ok {
loc, err := rang.getLocation(self.Database)
if err != nil {
return nil, err
}
step := RangeQueryStep{&BaseQueryStep{id, "range", loc, location}, rang.bitmap, rang.start, rang.end}
plan := QueryPlan{step}
return &plan, nil
} else if set, ok := qt.(*SetQueryTree); ok {
loc, err := set.getLocation(self.Database)
if err != nil {
return nil, err
}
step := SetQueryStep{&BaseQueryStep{id, "set", loc, location}, set.bitmap, set.profile_id}
plan := QueryPlan{step}
return &plan, nil
} else if clear, ok := qt.(*ClearQueryTree); ok {
loc, err := clear.getLocation(self.Database)
if err != nil {
return nil, err
}
step := ClearQueryStep{&BaseQueryStep{id, "clear", loc, location}, clear.bitmap, clear.profile_id}
plan := QueryPlan{step}
return &plan, nil
} else if cnt, ok := qt.(*CountQueryTree); ok {
sub_id := pilosa.NewGUID()
loc, err := cnt.getLocation(self.Database)
if err != nil {
return nil, err
}
step := &CountQueryStep{&BaseQueryStep{id, "count", loc, location}, &sub_id}
subq_steps, err := self.flatten(cnt.subquery, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
plan = append(plan, step)
} else if topn, ok := qt.(*TopNQueryTree); ok {
sub_id := pilosa.NewGUID()
loc, err := topn.getLocation(self.Database)
if err != nil {
return nil, err
}
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", loc, location}, nil, topn.Filters, topn.N, topn.Frame}
switch topn.subquery.(type) {
case *AllQueryTree:
// do nothing
default:
step.Input = &sub_id
subq_steps, err := self.flatten(topn.subquery, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
}
return &plan, nil
}
// Transforms Query into QueryTree and flattens to QueryPlan object
func (self *QueryPlanner) Plan(query *Query, id *pilosa.GUID, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanner.Plan", self, query, id, destination)
queryTree, err := self.buildTree(query, -1)
if err != nil {
return nil, err
}
return self.flatten(queryTree, query.Id, destination)
}
///////////////////////////////////////////////////////////////////////////////////////////////////
//MASK
///////////////////////////////////////////////////////////////////////////////////////////////////
type MaskQueryStep struct {
*BaseQueryStep
start, end uint64
}
type MaskQueryResult struct {
*BaseQueryResult
}
// QueryTree for Mask queries
type MaskQueryTree struct {
start, end uint64
bitmap *db.Bitmap
}
// Uses consistent hashing function to select node containing data for GET operation
/*
func (qt *MaskQueryTree) getLocation(d *db.Database) (*db.Location, error) {
slice := d.GetOrCreateSlice(qt.slice) // TODO: this should probably be just GetSlice (no create)
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("GetFragmenForBitmapFailed GetQueryTree", slice)
return nil, err
}
return fragment.GetLocation(), nil
}
*/
///////////////////////////////////////////////////////////////////////////////////////////////////
//Range
///////////////////////////////////////////////////////////////////////////////////////////////////
type RangeQueryStep struct {
*BaseQueryStep
Bitmap *db.Bitmap
Start, End time.Time
}
type RangeQueryResult struct {
*BaseQueryResult
}
// QueryTree for Mask queries
type RangeQueryTree struct {
bitmap *db.Bitmap
slice int
start, end time.Time
}
func (qt *RangeQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("RangeQueryTree", qt, d)
slice := d.GetOrCreateSlice(qt.slice) // TODO: this should probably be just GetSlice (no create)
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("GetFragmenForBitmapFailed GetQueryTree", slice)
return nil, err
}
return fragment.GetLocation(), nil
}
type FillResult struct {
*BaseQueryResult
}
///////////////////////////////////////////////////////////////////////////////////////////////////
//Stash
///////////////////////////////////////////////////////////////////////////////////////////////////
type CacheItem struct {
FragmentId pilosa.SUUID
Handle pilosa.BitmapHandle
}
type Stash struct {
Stash []CacheItem //pilosa.BitmapHandle //probably need to make the a struct with fragment_id and handle
incomplete bool
}
func NewStash() Stash {
return Stash{make([]CacheItem, 0), false}
}
func (st *Stash) Add(i pilosa.SUUID) {
item := CacheItem{i, 0}
st.Stash = append(st.Stash, item)
st.incomplete = true
}
func (st *Stash) Assign(i pilosa.BitmapHandle) {
st.Stash[len(st.Stash)-1].Handle = i //big assumption that item already exists
st.incomplete = false
}
type StashQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
N int
}
type StashQueryResult struct {
*BaseQueryResult
}
// QueryTree for UNION queries
type StashQueryTree struct {
subqueries []QueryTree
location *db.Location
N int
}
func (qt *StashQueryTree) Append(subtree QueryTree) {
qt.subqueries = append(qt.subqueries, subtree)
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *StashQueryTree) getLocation(d *db.Database) (*db.Location, error) {
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
type RecallQueryStep struct {
*BaseQueryStep
Stash Stash
}
type RecallQueryTree struct {
Stash Stash
}
func (rqt *RecallQueryTree) getLocation(d *db.Database) (*db.Location, error) {
return nil, nil
}
type RecallQueryResult struct {
*BaseQueryResult
}
type ClearQueryStep struct {
*BaseQueryStep
Bitmap *db.Bitmap
ProfileId uint64
}
type ClearQueryResult struct {
*BaseQueryResult
}
type ClearQueryTree struct {
bitmap *db.Bitmap
profile_id uint64
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *ClearQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("ClearQueryTree.getLocation", d)
// check here for supported frames
if !d.IsValidFrame(qt.bitmap.FrameType) {
return nil, NewInvalidFrame(d.Name, qt.bitmap.FrameType)
}
slice, err := d.GetSliceForProfile(qt.profile_id)
if err != nil {
return nil, NewFragmentNotFound(d.Name, qt.bitmap.FrameType, db.GetSlice(qt.profile_id))
}
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("NOT FOUND:", slice, qt.bitmap)
return nil, NewFragmentNotFound(d.Name, qt.bitmap.FrameType, db.GetSlice(qt.profile_id))
}
return fragment.GetLocation(), nil
}

67
query/planner_example.go
View file

@ -1,67 +0,0 @@
package query
//package main
//
//import (
// "pilosa/query"
// "pilosa/core"
// "math/rand"
// "time"
// "log"
//)
//
//func main() {
// rand.Seed(time.Now().UnixNano())
// cluster := core.Cluster{Self:"192.168.1.100:1201"}
// database := cluster.AddDatabase("property49")
// frame := database.AddFrame("general")
// frame.AddSlice("192.168.1.100:1201", "192.168.1.100:1202", "192.168.1.100:1203")
// frame.AddSlice("192.168.1.101:1201", "192.168.1.101:1202", "192.168.1.101:1203")
// frame.AddSlice("192.168.1.102:1201", "192.168.1.102:1202", "192.168.1.102:1203")
// frame2 := database.AddFrame("brands")
// frame2.AddSlice("192.168.1.200:1201", "192.168.1.200:1202", "192.168.1.200:1203")
// frame2.AddSlice("192.168.1.201:1201", "192.168.1.201:1202", "192.168.1.201:1203")
// frame2.AddSlice("192.168.1.202:1201", "192.168.1.202:1202", "192.168.1.202:1203")
//
// //cluster.describe()
//
// //query := Query{"get", []QueryInput{Bitmap{"general", 10}}}
//
// //query := Query{"union", []QueryInput{
// // &Query{"get", []QueryInput{Bitmap{"general", 20}}},
// // &Query{"get", []QueryInput{Bitmap{"brands", 30}}},
// //}}
//
// //queryString = "union(bitmap(general, 33), bitmap(brands, 44))"
// //queryString := `["union", ["intersect", ["bitmap", "general", 33], ["bitmap", "brands", 44]], ["bitmap", "general", 55]]`
// //query := `count(union(bitmap(""), bitmap(brands, 55)))`
// //query := `setbit(bitmap(cats, 33), 75000)`
// queryString := `["union", ["intersect", ["bitmap", "general", 33], ["bitmap", "brands", 44]], ["bitmap", "general", 55]]`
// queryParser := query.QueryParser{queryString}
// queryParsed, err := queryParser.Parse()
// log.Println(queryParsed)
// if err != nil {
// log.Println("ERROR!", err)
// }
//
//// query := qp.Query{"inter", []qp.QueryInput{
//// &qp.Query{"union", []qp.QueryInput{
//// &qp.Query{"get", []qp.QueryInput{core.Bitmap{"general", 20}}},
//// &qp.Query{"get", []qp.QueryInput{core.Bitmap{"brands", 30}}},
//// }},
//// &qp.Query{"get", []qp.QueryInput{core.Bitmap{"general", 10}}},
//// }}
////
// planner := query.QueryPlanner{&cluster, database}
//
// id, _ := uuid.NewV4()
// dest := "1.2.3.4:1234"
// log.Println("Query id", id, "Dest", dest)
//
// queryplan := planner.Plan(queryParsed, id, dest, -1)
//
// for _, i := range *queryplan {
// log.Println(i)
// }
// //fmt.Println(queryplan)
//}

283
query/planner_test.go
View file

@ -1,283 +0,0 @@
package query_test
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/query"
"github.com/umbel/pilosa/util"
)
// Ensure the query planner can plan a "get" query.
func TestQueryPlanner_Plan_Get(t *testing.T) {
q, err := query.QueryForPQL("get(10,default)")
if err != nil {
t.Fatal(err)
}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: q}
id := util.RandomUUID()
plan, err := planner.Plan(q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 3 {
t.Fatalf("unexpected plan length: %d", n)
}
if step := (*plan)[0].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if step.Slice != 0 {
t.Fatalf("unexpected step(0) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(0) bitmap: %s", spew.Sprint(step.Bitmap))
}
if step := (*plan)[1].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(1) operation: %s", step.Operation)
} else if step.Slice != 1 {
t.Fatalf("unexpected step(1) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(1) bitmap: %s", spew.Sprint(step.Bitmap))
}
if step := (*plan)[2].(query.CatQueryStep); step.Operation != "cat" {
t.Fatalf("unexpected step(2) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[0].(query.GetQueryStep).Id,
(*plan)[1].(query.GetQueryStep).Id,
}) {
t.Fatalf("unexpected step(2) inputs: %s", spew.Sprint(step.Inputs))
}
}
// Ensure the query planner can plan a "set" query.
func TestQueryPlanner_Plan_Set(t *testing.T) {
q, err := query.QueryForPQL("set(10, default, 0, 100)")
if err != nil {
t.Fatal(err)
}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: q}
id := util.RandomUUID()
plan, err := planner.Plan(q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 1 {
t.Fatalf("unexpected plan length: %d", n)
}
if step := (*plan)[0].(query.SetQueryStep); step.Operation != "set" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if step.ProfileId != 100 {
t.Fatalf("unexpected step(0) profile id: %d", step.ProfileId)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(0) bitmap: %s", spew.Sprint(step.Bitmap))
}
}
// Ensure the query planner can plan a "top-n" query.
func TestQueryPlanner_Plan_TopN(t *testing.T) {
q, err := query.QueryForPQL("top-n(get(10, default), default, 50,[1,2,3])")
if err != nil {
t.Fatal(err)
}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: q}
id := util.RandomUUID()
plan, err := planner.Plan(q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 5 {
t.Fatalf("unexpected plan length: %d", n)
}
if step := (*plan)[0].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(0) bitmap: %s", spew.Sprint(step.Bitmap))
}
if step := (*plan)[1].(*query.TopNQueryStep); step.Operation != "top-n" {
t.Fatalf("unexpected step(1) operation: %s", step.Operation)
} else if step.Input != (*plan)[0].(query.GetQueryStep).Id {
t.Fatalf("unexpected step(1) input: %d", step.Input)
} else if step.N != 50 {
t.Fatalf("unexpected step(1) n: %d", step.N)
} else if !reflect.DeepEqual(step.Filters, []uint64{1, 2, 3}) {
t.Fatalf("unexpected step(1) filters: %s", spew.Sprint(step.Filters))
}
if step := (*plan)[2].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(2) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(2) bitmap: %s", spew.Sprint(step.Bitmap))
}
if step := (*plan)[3].(*query.TopNQueryStep); step.Operation != "top-n" {
t.Fatalf("unexpected step(3) operation: %s", step.Operation)
} else if step.Input != (*plan)[2].(query.GetQueryStep).Id {
t.Fatalf("unexpected step(3) input: %d", step.Input)
} else if step.N != 50 {
t.Fatalf("unexpected step(3) n: %d", step.N)
}
}
// Ensure the query planner can plan a "top-n" all() query.
func TestQueryPlanner_Plan_TopN_All(t *testing.T) {
q, err := query.QueryForPQL("top-n(all(), default, 50, [1,2,3])")
if err != nil {
t.Fatal(err)
}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: q}
id := util.RandomUUID()
plan, err := planner.Plan(q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 3 {
t.Fatalf("unexpected plan length: %d", n)
}
if step := (*plan)[0].(*query.TopNQueryStep); step.Operation != "top-n" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if step.Input != nil {
t.Fatalf("unexpected step(0) input: %v", step.Input)
} else if step.N != 50 {
t.Fatalf("unexpected step(0) n: %d", step.N)
} else if !reflect.DeepEqual(step.Filters, []uint64{1, 2, 3}) {
t.Fatalf("unexpected step(0) filters: %s", spew.Sprint(step.Filters))
}
}
// Ensure the query planner can plan a "union" query.
func TestQueryPlanner_Plan_Union(t *testing.T) {
id1 := util.RandomUUID()
q1 := query.Query{Id: &id1, Operation: "get", Args: map[string]interface{}{"id": uint64(10), "frame": "default"}}
id2 := util.RandomUUID()
q2 := query.Query{Id: &id2, Operation: "get", Args: map[string]interface{}{"id": uint64(20), "frame": "default"}}
id3 := util.RandomUUID()
q := query.Query{Id: &id3, Operation: "union", Subqueries: []query.Query{q1, q2}}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: &q}
id := util.RandomUUID()
plan, err := planner.Plan(&q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 7 {
t.Fatalf("unexpected plan size: %d", n)
}
// First step should be a "get" step.
if step := (*plan)[0].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if step.Slice != 0 {
t.Fatalf("unexpected step(0) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(0) bitmap: %s", spew.Sprint(step.Bitmap))
}
// Second step should also be a "get" step.
if step := (*plan)[1].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(1) operation: %s", step.Operation)
} else if step.Slice != 0 {
t.Fatalf("unexpected step(1) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 20, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(1) bitmap: %s", spew.Sprint(step.Bitmap))
}
// Third step should union the first two steps.
if step := (*plan)[2].(query.UnionQueryStep); step.Operation != "union" {
t.Fatalf("unexpected step(2) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[0].(query.GetQueryStep).Id,
(*plan)[1].(query.GetQueryStep).Id,
}) {
t.Fatalf("unexpected step(2) inputs: %s", spew.Sprint(step.Inputs))
}
// Fourth step should be a "get" step.
if step := (*plan)[3].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(3) operation: %s", step.Operation)
} else if step.Slice != 1 {
t.Fatalf("unexpected step(3) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(3) bitmap: %s", spew.Sprint(step.Bitmap))
}
// Fifth step should also be a "get" step.
if step := (*plan)[4].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(4) operation: %s", step.Operation)
} else if step.Slice != 1 {
t.Fatalf("unexpected step(4) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 20, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(4) bitmap: %s", spew.Sprint(step.Bitmap))
}
// Sixth step should union the previous two steps.
if step := (*plan)[5].(query.UnionQueryStep); step.Operation != "union" {
t.Fatalf("unexpected step(5) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[3].(query.GetQueryStep).Id,
(*plan)[4].(query.GetQueryStep).Id,
}) {
t.Fatalf("unexpected step(5) inputs: %s", spew.Sprint(step.Inputs))
}
// Final step should concatenate the two union steps.
if step := (*plan)[6].(query.CatQueryStep); step.Operation != "cat" {
t.Fatalf("unexpected step(6) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[2].(query.UnionQueryStep).Id,
(*plan)[5].(query.UnionQueryStep).Id,
}) {
t.Fatalf("unexpected step(6) inputs: %s", spew.Sprint(step.Inputs))
}
}
// NewDefaultDB returns a simple, initialized database and fragment.
func NewDefaultDB() (*db.Database, *db.Fragment) {
// Create an empty database
c := db.NewCluster()
d := c.GetOrCreateDatabase("main")
f := d.GetOrCreateFrame("default")
s1 := d.GetOrCreateSlice(0)
frag1 := d.GetOrCreateFragment(f, s1, util.Id())
pid := util.RandomUUID()
p1 := db.NewProcess(&pid)
p1.SetHost("----192.1.1.0----")
frag1.SetProcess(p1)
s2 := d.GetOrCreateSlice(1)
frag2 := d.GetOrCreateFragment(f, s2, util.Id())
pid = util.RandomUUID()
p2 := db.NewProcess(&pid)
p2.SetHost("----192.1.1.1----")
frag2.SetProcess(p2)
return d, frag1
}

111
query/query.go
View file

@ -1,111 +0,0 @@
package query
import (
"strings"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
type QueryInput interface{}
type QueryResults struct {
Data interface{}
}
type PqlList []PqlListItem
type PqlListItem struct {
Id *pilosa.GUID
Label string
PQL string
}
type Query struct {
Id *pilosa.GUID
Operation string
Args map[string]interface{}
Subqueries []Query
}
func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanFOrPQL", database, pql, destination)
tokens, err := Lex(pql)
if err != nil {
return nil, err
}
return QueryPlanForTokens(database, tokens, destination)
}
func QueryForPQL(pql string) (*Query, error) {
log.Trace("QueryForPQL", pql)
tokens, err := Lex(pql)
if err != nil {
return nil, err
}
return QueryForTokens(tokens)
}
func QueryForTokens(tokens []Token) (*Query, error) {
log.Trace("QueryForTokens", tokens)
query, err := Parse(tokens)
if err != nil {
return nil, err
}
return query, nil
}
func QueryPlanForTokens(database *db.Database, tokens []Token, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanForTokens", database, tokens, destination)
query, err := QueryForTokens(tokens)
if err != nil {
return nil, err
}
return QueryPlanForQuery(database, query, destination)
}
func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanForQuery", database, query, destination)
query_planner := QueryPlanner{Database: database, Query: query}
id := pilosa.NewGUID()
query_plan, err := query_planner.Plan(query, &id, destination)
if err != nil {
return nil, err
}
return query_plan, nil
}
func TokensToFilterStrings(tokens []Token) (string, []string) {
log.Trace("TokensToFilterStrings", tokens)
var whole []string
var filter string
var filters []string
var open_parens int
var in_square_brackets bool
var last_slice = 0
open_parens = -1
in_square_brackets = false
for i, _ := range tokens {
whole = append(whole, tokens[i].Text)
if tokens[i].Type == TYPE_FUNC {
last_slice = i
} else if tokens[i].Type == TYPE_LP {
open_parens += 1
} else if tokens[i].Type == TYPE_RP {
open_parens -= 1
if open_parens == 0 {
if !in_square_brackets {
filter = strings.Join(whole[2:], "")
} else {
filters = append(filters, strings.Join(whole[last_slice:], ""))
}
last_slice = i
}
} else if open_parens == 0 && tokens[i].Type == TYPE_LB {
in_square_brackets = true
}
}
return filter, filters
}

36
query/query_test.go
View file

@ -1,36 +0,0 @@
package query_test
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/query"
)
func TestTokensToFilterStrings1(t *testing.T) {
filter, filters := query.TokensToFilterStrings(MustLex("plugin(get(88, general), [get(12, general), get(13, general)])"))
if filter != "get(88,general)" {
t.Fatalf("unexpected filter: %s", filter)
} else if !reflect.DeepEqual(filters, []string{"get(12,general)", "get(13,general)"}) {
t.Fatalf("unexpected filters: %s", spew.Sprint(filters))
}
}
func TestTokensToFilterStrings2(t *testing.T) {
filter, filters := query.TokensToFilterStrings(MustLex("plugin(intersect(get(88, general, [0]), get(77, b.n)), [get(12, general), get(13, general)])"))
if filter != "intersect(get(88,general,[0]),get(77,b.n))" {
t.Fatalf("unexpected filter: %s", filter)
} else if !reflect.DeepEqual(filters, []string{"get(12,general)", "get(13,general)"}) {
t.Fatalf("unexpected filters: %s", spew.Sprint(filters))
}
}
func TestTokensToFilterStrings3(t *testing.T) {
filter, filters := query.TokensToFilterStrings(MustLex("plugin(intersect(get(88, general, [0]), get(77, b.n)))"))
if filter != "intersect(get(88,general,[0]),get(77,b.n))" {
t.Fatalf("unexpected filter: %s", filter)
} else if !reflect.DeepEqual(filters, []string(nil)) {
t.Fatalf("unexpected filters: %s", spew.Sprint(filters))
}
}

65
statsd/statd.go
View file

@ -1,65 +0,0 @@
package statsd
import (
"time"
"github.com/cactus/go-statsd-client/statsd"
log "github.com/cihub/seelog"
)
// DefaultHost is the default host to send statsd data to.
const DefaultHost = "127.0.0.1:8125"
// Host is the host to send statsd data to.
var Host = DefaultHost
type args struct {
stat string
delta int64
rate float32
}
var timer chan args
var count chan string
var end chan bool
func Setup() {
timer = make(chan args, 32768)
count = make(chan string, 32768)
end = make(chan bool)
log.Warn("New Stats", Host)
stats, _ := statsd.New(Host, "")
go func() {
for {
select {
case ci := <-timer:
stats.Gauge(ci.stat, ci.delta, ci.rate)
stats.Timing(ci.stat, ci.delta, ci.rate)
case stat := <-count:
stats.Inc(stat, 1, 1.0)
case <-end:
log.Warn("DONE Stats")
return
}
}
}()
}
func SendTimer(stat string, delta int64) {
pstat := "pilosa." + stat
milli := time.Duration(delta) / time.Millisecond
timer <- args{pstat, int64(milli), 1.0}
}
func SendInc(stat string) {
pstat := "pilosa." + stat
count <- pstat
}
func ShutdownStats() {
log.Warn("Shutdown Stats")
end <- true
}

49
storage.go
View file

@ -1,49 +0,0 @@
package pilosa
// Storage represents
type Storage interface {
Open() error
Close() error
Flush()
Fetch(bitmap_id uint64, db, frame string, slice int) (*Bitmap, uint64)
Store(id uint64, db, frame string, slice int, filter uint64, bitmap *Bitmap) error
StoreBlock(id uint64, db, frame string, slice int, filter uint64, chunk uint64, block_index int32, block uint64) error
RemoveBlock(id uint64, db, frame string, slice int, chunk uint64, block_index int32)
StoreBit(bid uint64, db, frame string, slice int, filter uint64, chunk uint64, block_index int32, block, count uint64)
RemoveBit(id uint64, db, frame string, slice int, filter uint64, chunk uint64, block_index int32, count uint64)
}
var storageFns = make(map[string]NewStorageFunc)
// NewStorageFunc represents a function that instantiates a new storage engine.
type NewStorageFunc func(opt StorageOptions) Storage
// RegisterStorage registers a storage engine.
func RegisterStorage(name string, fn NewStorageFunc) {
if storageFns[name] != nil {
panic("storage engine already registered: " + name)
}
storageFns[name] = fn
}
// NewStorage returns a new Storage instance by name.
func NewStorage(name string, opt StorageOptions) Storage {
fn := storageFns[name]
if fn == nil {
panic("storage type not registered: " + name)
}
return fn(opt)
}
// StorageOptions represents the options passed to the storage engine.
type StorageOptions struct {
DB string
Slice int
Frame string
FragmentID SUUID
LevelDBPath string
}

244
storage/cassandra/cassandra.go
View file

@ -1,244 +0,0 @@
package cassandra
import (
"time"
log "github.com/cihub/seelog"
"github.com/gocql/gocql"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
)
func init() {
pilosa.RegisterStorage("cassandra",
func(opt pilosa.StorageOptions) pilosa.Storage {
return NewStorage(opt)
},
)
}
// CREATE KEYSPACE IF NOT EXISTS pilosa WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1"
// create keyspace if not exists pilosa with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true;
// CREATE KEYSPACE pilosa WITH replication = {'class': 'NetworkTopologyStrategy', 'pilpang': '2'} AND durable_writes = true;
// CREATE TABLE IF NOT EXISTS bitmap (bitmap_id bigint, db varchar, frame varchar, slice int, filter int, chunkkey bigint, blockindex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame, slice), chunkkey, blockindex) )
// DefaultHosts are the default hosts in the cassandra cluster.
var DefaultHosts = []string{"localhost"}
const (
// DefaultKeyspace is the default keyspace used in cassandra.
DefaultKeyspace = "pilosa"
// DefaultFlushInterval is the default maximum time between flushes.
DefaultFlushInterval = 5 * time.Second
// DefaultFlushThreshold is the default maximum number of items to batch.
DefaultFlushThreshold = 15
)
// Storage represents Cassandra-backed storage for bitmaps.
type Storage struct {
session *gocql.Session
batch *gocql.Batch
batchTime time.Time
batchN int
FlushInterval time.Duration
FlushThreshold int
Hosts []string
Keyspace string
}
// NewStorage returns a new, uninitialized instance of Storage.
func NewStorage(opt pilosa.StorageOptions) *Storage {
return &Storage{
batchTime: time.Now(),
FlushInterval: DefaultFlushInterval,
FlushThreshold: DefaultFlushThreshold,
Hosts: DefaultHosts,
Keyspace: DefaultKeyspace,
}
}
// Open opens the connection to the cassandra cluster.
func (s *Storage) Open() error {
// Create cluster configuration.
config := gocql.NewCluster(s.Hosts...)
config.Keyspace = s.Keyspace
config.Consistency = gocql.One
config.Timeout = 5 * time.Second
config.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: 10}
// Connect to cassandra.
session, err := config.CreateSession()
if err != nil {
return err
}
s.session = session
return nil
}
// Close closes the connection to the cassandra cluster.
func (s *Storage) Close() error {
if s.session != nil {
s.Flush()
s.session.Close()
}
return nil
}
// Fetch returns a bitmap by ID.
func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*pilosa.Bitmap, uint64) {
bm := pilosa.NewBitmap()
// Start benchmark.
start := time.Now()
// Create iterator over bitmap.
itr := s.session.Query("SELECT filter, Chunkkey, BlockIndex, block FROM bitmap WHERE bitmap_id=? AND db=? AND frame=? AND slice=? ",
u64toi64(bitmapID), db, frame, slice).Iter()
// Iterate over chunks and materialize bitmap object.
var chunk *pilosa.Chunk
var chunkKey, block, count int64
var blockIndex uint32
var filter int
lastKey := int64(-1)
for itr.Scan(&filter, &chunkKey, &blockIndex, &block) {
if chunkKey != int64(-1) {
if chunkKey != lastKey {
chunk = &pilosa.Chunk{uint64(chunkKey), pilosa.NewBlocks()}
bm.AddChunk(chunk)
}
chunk.Value[uint8(blockIndex)] = uint64(block)
} else {
count = block
}
lastKey = chunkKey
}
statsd.SendTimer("cassandra_storage_Fetch", time.Since(start).Nanoseconds())
statsd.SendInc("cassandra_storage_Read")
// Set total bits set.
bm.SetCount(uint64(count))
return bm, uint64(filter)
}
// beginBatch starts a batch if one is not already in progress.
func (s *Storage) beginBatch() {
if s.batch == nil {
s.batch = gocql.NewBatch(gocql.UnloggedBatch)
}
s.batchN++
}
// endBatch flushes a batch if one is in progress.
func (s *Storage) endBatch() {
if s.batch == nil {
return
}
start := time.Now()
s.Flush()
statsd.SendTimer("cassandra_storage_EndBatch", time.Since(start).Nanoseconds())
statsd.SendInc("cassandra_storage_Write")
}
// Flush flushes the current batch to storage.
func (s *Storage) Flush() {
start := time.Now()
// If batch exists then flush it.
if s.batch != nil {
if err := s.session.ExecuteBatch(s.batch); err != nil {
log.Warn("Batch ERROR: ", err)
}
}
// Clear batch and threshold time and count.
s.batch = nil
s.batchTime = time.Now()
s.batchN = 0
statsd.SendTimer("cassandra_storage_FlushBatch", time.Since(start).Nanoseconds())
}
// Store saves a bitmap to storage.
func (s *Storage) Store(id uint64, db string, frame string, slice int, filter uint64, bm *pilosa.Bitmap) error {
s.beginBatch()
for i := bm.ChunkIterator(); !i.Limit(); i = i.Next() {
var chunk = i.Item()
for idx, block := range chunk.Value {
if block != 0 {
s.StoreBlock(id, db, frame, slice, filter, chunk.Key, int32(idx), block)
}
}
}
s.StoreBlock(id, db, frame, slice, filter, pilosa.CounterMask, 0, bm.BitCount())
s.endBatch()
return nil
}
// StoreBlock saves a block to storage.
func (s *Storage) StoreBlock(bid uint64, db string, frame string, slice int, filter uint64, bchunk uint64, blockIndex int32, bblock uint64) error {
id := u64toi64(bid) // these functions ignore overflow
block := u64toi64(bblock)
chunk := u64toi64(bchunk)
if s.batch == nil {
s.beginBatch()
}
start := time.Now()
s.batch.Query(`INSERT INTO bitmap ( bitmap_id, db, frame, slice , filter, ChunkKey, BlockIndex, block) VALUES (?,?,?,?,?,?,?,?) USING timestamp ?;`,
id, db, frame, slice, int(filter), chunk, blockIndex, block, start.UnixNano())
statsd.SendTimer("cassandra_storage_StoreBlock", time.Since(start).Nanoseconds())
return nil
}
// RemoveBlock deletes a block from storage.
func (s *Storage) RemoveBlock(bid uint64, db string, frame string, slice int, bchunk uint64, blockIndex int32) {
log.Trace("RemoveBlock", bid, db, frame, slice, bchunk, blockIndex)
id := u64toi64(bid) //these fucntions ignore overflow
chunk := u64toi64(bchunk)
if s.batch == nil {
s.beginBatch()
}
startTime := time.Now()
s.batch.Query(`DELETE FROM bitmap USING TIMESTAMP ? WHERE bitmap_id=? AND db=? AND frame=? AND slice=? AND chunkkey=? AND blockindex=?`,
startTime.UnixNano(), id, db, frame, slice, chunk, blockIndex)
statsd.SendTimer("cassandra_storage_DeleteBlock", time.Since(startTime).Nanoseconds())
}
func (s *Storage) StoreBit(bid uint64, db string, frame string, slice int, filter uint64, chunk uint64, blockIndex int32, val, count uint64) {
s.beginBatch()
s.StoreBlock(bid, db, frame, slice, filter, chunk, blockIndex, val)
s.StoreBlock(bid, db, frame, slice, filter, pilosa.CounterMask, 0, count)
s.endBatch()
}
func (s *Storage) RemoveBit(id uint64, db string, frame string, slice int, filter uint64, chunk uint64, blockIndex int32, count uint64) {
log.Trace("RemoveBit", id, db, frame, slice, chunk, blockIndex)
s.beginBatch()
s.RemoveBlock(id, db, frame, slice, chunk, blockIndex)
s.StoreBlock(id, db, frame, slice, filter, pilosa.CounterMask, 0, count)
s.endBatch()
}
func u64toi64(v uint64) int64 { return int64(v) }

1
storage/cassandra/cassandra_test.go
View file

@ -1 +0,0 @@
package cassandra_test

222
storage/leveldb/leveldb.go
View file

@ -1,222 +0,0 @@
package leveldb
import (
"bytes"
"encoding/binary"
"path/filepath"
"strconv"
"time"
"github.com/syndtr/goleveldb/leveldb"
. "github.com/syndtr/goleveldb/leveldb/util"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
)
func init() {
pilosa.RegisterStorage("leveldb",
func(opt pilosa.StorageOptions) pilosa.Storage {
return NewStorage(opt)
},
)
}
//go get github.com/syndtr/goleveldb/leveldb
// Storage represents a LevelDB-backed storage engine.
type Storage struct {
path string
db *leveldb.DB
batch *leveldb.Batch
batchTime time.Time
batchN int
}
// NewStorage returns a new instance of Storage.
func NewStorage(opt pilosa.StorageOptions) *Storage {
path := filepath.Join(
opt.LevelDBPath,
opt.DB,
strconv.Itoa(opt.Slice),
opt.Frame,
opt.FragmentID.String(),
)
return &Storage{path: path}
}
// Path returns the path the storage was initialized with.
func (s *Storage) Path() string { return s.path }
// Open opens and initializes the storage.
func (s *Storage) Open() error {
db, err := leveldb.OpenFile(s.path, nil)
if err != nil {
return err
}
s.db = db
s.batchTime = time.Now().Add(-time.Hour)
return nil
}
// Close flushes and closes the storage.
func (s *Storage) Close() error {
s.Flush()
return s.db.Close()
}
func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*pilosa.Bitmap, uint64) {
bm := pilosa.NewBitmap()
// Begin benchmark.
start := time.Now()
// Create an iterator on the database.
iter := s.db.NewIterator(&Range{
Start: marshalKey(bitmapID, 0, 0),
Limit: marshalKey(bitmapID+1, 0, 0),
}, nil)
defer iter.Release()
// Iterate over blocks in database and create bitmap.
var chunk *pilosa.Chunk
var filter, block, count uint64
lastKey := uint64(pilosa.CounterMask)
for iter.Next() {
_, key, idx := unmarshalKey(iter.Key())
block, filter = unmarshalValue(iter.Value())
if key != pilosa.CounterMask {
if key != lastKey {
chunk = &pilosa.Chunk{key, pilosa.NewBlocks()}
bm.AddChunk(chunk)
}
chunk.Value[idx] = block
} else {
count = block
}
lastKey = key
}
statsd.SendTimer("leveldb_storage_Fetch", time.Since(start).Nanoseconds())
// Set bit count on bitmap.
bm.SetCount(uint64(count))
return bm, uint64(filter)
}
// beginBatch starts a new batch if one is not already started.
func (s *Storage) beginBatch() {
if s.batch == nil {
s.batch = &leveldb.Batch{}
}
s.batchN++
}
func (s *Storage) endBatch() {
if s.batch == nil {
return
}
start := time.Now()
if time.Since(s.batchTime) > 15*time.Second || s.batchN > 300 {
s.Flush()
}
statsd.SendTimer("leveldb_storage_EndBatch", time.Since(start).Nanoseconds())
}
func (s *Storage) Flush() {
start := time.Now()
// Flush the batch if one exists.
if s.batch != nil {
s.db.Write(s.batch, nil)
}
// Clear batch and reset timer and count.
s.batch = nil
s.batchTime = time.Now()
s.batchN = 0
statsd.SendTimer("leveldb_storage_FlushBatch", time.Since(start).Nanoseconds())
}
// Store saves a bitmap to storage.
func (s *Storage) Store(bitmapID uint64, db string, frame string, slice int, filter uint64, bm *pilosa.Bitmap) error {
s.beginBatch()
for itr := bm.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
for idx, block := range itr.Item().Value {
if block != 0 {
s.StoreBlock(bitmapID, db, frame, slice, filter, itr.Item().Key, int32(idx), block)
}
}
}
s.StoreBlock(bitmapID, db, frame, slice, filter, pilosa.CounterMask, 0, bm.BitCount())
s.endBatch()
return nil
}
// StoreBlock saves a block to storage.
func (s *Storage) StoreBlock(bitmapID uint64, db string, frame string, slice int, filter uint64, chunk uint64, index int32, block uint64) error {
start := time.Now()
s.batch.Put(marshalKey(bitmapID, chunk, uint8(index)), marshalValue(block, filter))
statsd.SendTimer("leveldb_storage_StoreBlock", time.Since(start).Nanoseconds())
return nil
}
// RemoveBlock deletes a block from storage. This is a no-op in the LevelDB store.
func (s *Storage) RemoveBlock(bitmapID uint64, db string, frame string, slice int, chunk uint64, blockIndex int32) {
}
// StoreBit sets a bit in storage.
func (s *Storage) StoreBit(bitmapID uint64, db string, frame string, slice int, filter uint64, bchunk uint64, blockIndex int32, bblock, count uint64) {
s.beginBatch()
s.StoreBlock(bitmapID, db, frame, slice, filter, bchunk, blockIndex, bblock)
s.StoreBlock(bitmapID, db, frame, slice, filter, pilosa.CounterMask, 0, count)
s.endBatch()
}
// RemoveBlock unsets a bit in storage. This is a no-op in the LevelDB store.
func (s *Storage) RemoveBit(bitmapID uint64, db string, frame string, slice int, filter uint64, bchunk uint64, blockIndex int32, count uint64) {
}
// marshalKey encodes the id, chunk key, & block index to a byte slice.
func marshalKey(id, key uint64, index uint8) []byte {
buf := new(bytes.Buffer)
binary.Write(buf, binary.LittleEndian, id)
binary.Write(buf, binary.LittleEndian, key)
binary.Write(buf, binary.LittleEndian, index)
return buf.Bytes()
}
// marshalValue encodes the block number and filter to a byte slice.
func marshalValue(block, filter uint64) []byte {
buf := new(bytes.Buffer)
binary.Write(buf, binary.LittleEndian, block)
binary.Write(buf, binary.LittleEndian, filter)
return buf.Bytes()
}
// unmarshalKey decodes the id, chunk key, & block index from a byte slice.
func unmarshalKey(v []byte) (id, key uint64, index uint8) {
buf := bytes.NewReader(v)
binary.Read(buf, binary.LittleEndian, &id)
binary.Read(buf, binary.LittleEndian, &key)
binary.Read(buf, binary.LittleEndian, &index)
return
}
// unmarshalValue decodes the block number and filter from a byte slice.
func unmarshalValue(v []byte) (block, filter uint64) {
buf := bytes.NewReader(v)
binary.Read(buf, binary.LittleEndian, &block)
binary.Read(buf, binary.LittleEndian, &filter)
return
}

55
storage/leveldb/leveldb_test.go
View file

@ -1,55 +0,0 @@
package leveldb_test
import (
"io/ioutil"
"os"
"testing"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/storage/leveldb"
"github.com/umbel/pilosa/util"
)
func init() {
util.SetupStatsd()
}
func TestStorage_Fetch(t *testing.T) {
s := MustOpenStorage()
defer s.Close()
}
// Storage represents a test wrapper for leveldb.Storage.
type Storage struct {
*leveldb.Storage
}
// NewStorage returns a new instance of Storage with a temporary path.
func NewStorage() *Storage {
// Create temporary path.
f, err := ioutil.TempFile("", "pilosa-leveldb-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return &Storage{leveldb.NewStorage(pilosa.StorageOptions{
LevelDBPath: f.Name(),
})}
}
// MustOpenStorage returns a new, opened instance of Storage.
func MustOpenStorage() *Storage {
s := NewStorage()
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the storage and removes the underlying data file.
func (s *Storage) Close() error {
defer os.Remove(s.Path())
return s.Storage.Close()
}

79
storage/mem/mem.go
View file

@ -1,79 +0,0 @@
package mem
import (
"fmt"
"github.com/umbel/pilosa"
)
func init() {
pilosa.RegisterStorage("memory",
func(opt pilosa.StorageOptions) pilosa.Storage {
return NewStorage()
},
)
}
// Storage represents in-memory bitmap storage.
type Storage struct {
db map[string]*pilosa.Bitmap
}
// NewStorage returns a new instance of Storage.
func NewStorage() *Storage {
return &Storage{
db: make(map[string]*pilosa.Bitmap),
}
}
// Open initializes the storage.
func (c *Storage) Open() error { return nil }
// Close shuts down the storage.
func (c *Storage) Close() error { return nil }
// FlushBatch flushes the batch to disk. This is a no-op for in-memory storage.
func (c *Storage) Flush() {}
// Fetch retrieves a bitmap by id.
func (c *Storage) Fetch(id uint64, db, frame string, slice int) (*pilosa.Bitmap, uint64) {
key := fmt.Sprintf("%d:%s:%s:%d", id, db, frame, slice)
// Find bitmap by key.
b, ok := c.db[key]
if ok {
return b, 0
}
// If the bitmap doesn't exist then create a new one.
b = pilosa.NewBitmap()
c.db[key] = b
return b, 0
}
// Store saves a bitmap to storage.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (c *Storage) Store(id uint64, db, frame string, slice int, filter uint64, b *pilosa.Bitmap) error {
return nil
}
// StoreBlock saves a block to storage.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (c *Storage) StoreBlock(id uint64, db, frame string, slice int, filter uint64, chunk_key uint64, block_index int32, block uint64) error {
return nil
}
// RemoveBlock deletes a block from bitmap.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (self *Storage) RemoveBlock(id uint64, db string, frame string, slice int, chunk uint64, block_index int32) {
}
// StoreBit sets a bit in a bitmap.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (self *Storage) StoreBit(id uint64, db string, frame string, slice int, filter uint64, bchunk uint64, block_index int32, bblock, count uint64) {
}
// RemoveBit unsets a bit in a bitmap.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (self *Storage) RemoveBit(id uint64, db string, frame string, slice int, filter uint64, chunk uint64, block_index int32, count uint64) {
}

30
storage/mem/mem_test.go
View file

@ -1,30 +0,0 @@
package mem_test
import (
"testing"
"github.com/umbel/pilosa/storage/mem"
)
// Ensure a bitmap can be retrieved from storage.
func TestStorage_Fetch(t *testing.T) {
s := mem.NewStorage()
// Retrieving a non-existent bitmap should create a new one.
b, _ := s.Fetch(1, "d", "f", 0)
if b == nil {
t.Fatal("expected bitmap")
}
// Retrieve the bitmap again.
other, _ := s.Fetch(1, "d", "f", 0)
if b != other {
t.Fatal("expected same bitmap")
}
// Retrieving a different bitmap should return a different reference.
b2, _ := s.Fetch(2, "d", "f", 0)
if b == b2 {
t.Fatal("expected new bitmap")
}
}

7
storage/storage.go
View file

@ -1,7 +0,0 @@
package storage
import (
_ "github.com/umbel/pilosa/storage/cassandra"
_ "github.com/umbel/pilosa/storage/leveldb"
_ "github.com/umbel/pilosa/storage/mem"
)

5
timeframe.go
View file

@ -86,15 +86,18 @@ func NextMonth(start time.Time, end time.Time) bool {
nextMonth := start.AddDate(0, 1, 0)
return sameMonth(nextMonth, end) || end.After(nextMonth)
}
func sameDay(t1, t2 time.Time) bool {
y1, m1, d1 := t1.Date()
y2, m2, d2 := t2.Date()
return (y1 == y2) && (m1 == m2) && (d1 == d2)
}
func NextDay(start time.Time, end time.Time) bool {
nextDay := start.AddDate(0, 0, 1)
return sameDay(nextDay, end) || end.After(nextDay)
}
func GetRange(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 {
results, marker := upHill(start_time, end_time, tile_id)
r2 := downHill(marker, end_time, tile_id)
@ -142,8 +145,8 @@ func upHill(start_time time.Time, end_time time.Time, tile_id uint64) ([]uint64,
return results, time_iterator
}
func downHill(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 {
func downHill(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 {
var results []uint64
time_iterator := start_time
for time_iterator.Before(end_time) {

56
transport/http.go
View file

@ -1,56 +0,0 @@
package transport
import (
"log"
"github.com/umbel/pilosa/db"
)
const DefaultHTTPPort = 15001
type HttpTransport struct {
port int
outbox chan *db.Message
done chan int
}
func (trans *HttpTransport) Init() error {
log.Println("Bind to port", trans.port)
trans.done = make(chan int)
go trans.Loop()
return nil
}
func (trans *HttpTransport) Loop() {
var message *db.Message
for {
select {
case message = <-trans.outbox:
log.Println(message)
case <-trans.done:
return
}
}
}
func (trans *HttpTransport) Close() {
log.Println("Closing HTTP transport.")
trans.done <- 1
}
func (trans *HttpTransport) Send(node string, message *db.Message) error {
log.Println("Send", message, "to", node)
trans.outbox <- message
return nil
}
func (trans *HttpTransport) Receive() (*db.Message, error) {
return &db.Message{}, nil
}
func NewHttpTransport(port int) *HttpTransport {
trans := new(HttpTransport)
trans.port = port
trans.outbox = make(chan *db.Message, 10)
return trans
}

187
transport/tcp.go
View file

@ -1,187 +0,0 @@
package transport
import (
"encoding/gob"
"fmt"
"net"
"os"
"time"
notify "github.com/bitly/go-notify"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
)
const DefaultTCPPort = 12001
type connection struct {
transport *TcpTransport
inbox chan *db.Message
outbox chan *db.Message
conn *net.Conn
process *pilosa.GUID
}
type newconnection struct {
id *pilosa.GUID
connection *connection
}
func init() {
gob.Register(pilosa.GUID{})
}
func (self *connection) manage() {
BeginManageConnection:
for {
if self.conn == nil {
process, err := self.transport.ProcessMap.GetProcess(self.process)
if err != nil {
log.Warn("transport/tcp: error getting process, retrying in 2 seconds... ", self.process, err)
time.Sleep(2 * time.Second)
continue
}
host_string := fmt.Sprintf("%s:%d", process.Host(), process.PortTcp())
conn, err := net.Dial("tcp", host_string)
if err != nil {
log.Warn("transport/tcp: error dialing: ", host_string, " Retrying in 2 seconds...")
time.Sleep(2 * time.Second)
continue
}
self.conn = &conn
go func() {
self.outbox <- &db.Message{Data: self.transport.ID.String()}
}()
}
encoder := gob.NewEncoder(*self.conn)
decoder := gob.NewDecoder(*self.conn)
var exit = make(chan int)
go func() {
for {
var mess *db.Message
err := decoder.Decode(&mess)
if err != nil {
log.Warn("transport/tcp: error decoding message: ", err.Error())
exit <- 1
return
}
self.inbox <- mess
}
}()
for {
select {
case message := <-self.outbox:
err := encoder.Encode(message)
if err != nil {
log.Warn(err.Error())
return
}
case message := <-self.inbox:
identifier, ok := message.Data.(pilosa.GUID)
if ok {
// message is connection registration; bypass inbox and register
self.process = &identifier
self.transport.reg <- &newconnection{&identifier, self}
} else {
self.transport.inbox <- message
}
case <-exit:
if self.process != nil {
self.conn = nil
continue BeginManageConnection
} else {
return
}
}
}
}
}
type TcpTransport struct {
inbox chan *db.Message
outbox chan db.Envelope
connections map[pilosa.GUID]*connection
reg chan *newconnection
ID pilosa.GUID
Port int
ProcessMap *core.ProcessMap
}
func NewTcpTransport(id pilosa.GUID) *TcpTransport {
return &TcpTransport{
inbox: make(chan *db.Message, 100),
outbox: make(chan db.Envelope, 100),
connections: make(map[pilosa.GUID]*connection),
reg: make(chan *newconnection),
ID: id,
Port: DefaultTCPPort,
}
}
func (self *TcpTransport) Run() {
log.Warn("Initializing TCP transport")
go self.listen()
for {
select {
case env := <-self.outbox:
con, ok := self.connections[*(env.Host)]
if !ok {
con = &connection{self, make(chan *db.Message, 100), make(chan *db.Message, 100), nil, env.Host}
go con.manage()
self.connections[*env.Host] = con
}
con.outbox <- env.Message
case nc := <-self.reg:
self.connections[*nc.id] = nc.connection
}
}
}
func (self *TcpTransport) listen() {
port_string := fmt.Sprintf(":%d", self.Port)
l, e := net.Listen("tcp", port_string)
if e != nil {
log.Critical("Cannot bind to port! ", self.Port)
os.Exit(-1)
}
for {
conn, err := l.Accept()
if err != nil {
log.Warn("Error accepting, trying again in 2 sec... ", err)
time.Sleep(2 * time.Second)
continue
}
go self.manage(&conn)
}
}
func (self *TcpTransport) manage(conn *net.Conn) {
con := &connection{self, make(chan *db.Message, 1024), make(chan *db.Message, 1024), conn, nil}
con.manage()
}
func (self *TcpTransport) Close() {
log.Warn("Shutting down TCP transport")
}
func (self *TcpTransport) Send(message *db.Message, host *pilosa.GUID) {
log.Trace("TcpTransport.Send", message, host)
envelope := db.Envelope{Message: message, Host: host}
notify.Post("outbox", &envelope)
self.outbox <- envelope
}
func (self *TcpTransport) Receive() *db.Message {
log.Trace("TcpTransport.Receive")
message := <-self.inbox
notify.Post("inbox", message)
return message
}
func (self *TcpTransport) Push(message *db.Message) {
self.inbox <- message
}

27
util.go
View file

@ -1,27 +0,0 @@
package pilosa
import (
"io"
"os"
"strings"
"github.com/kr/s3/s3util"
)
func openFile(s string) (io.ReadCloser, error) {
if isURL(s) {
return s3util.Open(s, nil)
}
return os.Open(s)
}
func createFile(s string) (io.WriteCloser, error) {
if isURL(s) {
return s3util.Create(s, nil, nil)
}
return os.Create(s)
}
func isURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
}