scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-05 16:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

refactor

Browse source 
This commit refactors the pilosa codebase. It makes several major
changes:

* Removes bitmap handles
* Removes dispatch/hold/transport
* Removes etcd dependency
* Adds consistent hash ring for slice placement
* Refactors parser/lexer
* Adds strong typing to PQL AST
* Flattens package hierarchy
This commit is contained in:
Ben Johnson 2015-09-23 17:23:38 -06:00
parent e600c22089
commit 34fa1afad0
503 changed files with 12764 additions and 127466 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

52
Godeps/Godeps.json generated
View file

@ -1,6 +1,6 @@
{
"ImportPath": "github.com/umbel/pilosa",
"GoVersion": "go1.5",
"GoVersion": "go1.5.1",
"Packages": [
"./..."
],
@ -10,67 +10,21 @@
"Comment": "v0.1.0-21-g056c9bc",
"Rev": "056c9bc7be7190eaa7715723883caffa5f8fa3e4"
},
{
"ImportPath": "github.com/bitly/go-notify",
"Rev": "0a148b8111d688ba7550fc7119fe0d5d8e650838"
},
{
"ImportPath": "github.com/cactus/go-statsd-client/statsd",
"Rev": "3999011ef0451eb805b0fa31f98cfb9261bd71be"
},
{
"ImportPath": "github.com/cihub/seelog",
"Comment": "go1.1-81-gc510775",
"Rev": "c510775bb50d98213cfafca75a4bc5e3fddc8d8f"
},
{
"ImportPath": "github.com/coreos/go-etcd/etcd",
"Rev": "1e26d8ee84cf9b1000d2af8acfb45b2521f49be5"
},
{
"ImportPath": "github.com/davecgh/go-spew/spew",
"Rev": "e762b3d1320b76030bd7f6cc2bfc3d9acce874c0"
},
{
"ImportPath": "github.com/gocql/gocql",
"Comment": "1st_gen_framing-221-gf8fb76b",
"Rev": "f8fb76bb772442ea938e4be46ab1666e542411a5"
"ImportPath": "github.com/gogo/protobuf/proto",
"Rev": "499788908625f4d83de42a204d1350fde8588e4f"
},
{
"ImportPath": "github.com/golang/groupcache/lru",
"Rev": "d781998583680cda80cf61e0b37dd0cd8da2eb52"
},
{
"ImportPath": "github.com/golang/snappy",
"Rev": "723cc1e459b8eea2dea4583200fd60757d40097a"
},
{
"ImportPath": "github.com/gorilla/websocket",
"Rev": "92334662baa9cbebc2e6e68b8d56bc1233f85a4c"
},
{
"ImportPath": "github.com/kr/s3",
"Rev": "c070c8f9a8f0032d48f0d2a77d4e382788bd8a1d"
},
{
"ImportPath": "github.com/robertkrimen/otto",
"Rev": "f9e07770bd9b5142de517b05a85ffbb42d1895da"
},
{
"ImportPath": "github.com/stathat/consistent",
"Rev": "0262985f12333852b6423025aa3ca315af473a4a"
},
{
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
"Rev": "183614d6b32571e867df4cf086f5480ceefbdfac"
},
{
"ImportPath": "github.com/yasushi-saito/rbtree",
"Rev": "571e2538414bf914c7e2909b61217b4e3e5508f4"
},
{
"ImportPath": "gopkg.in/inf.v0",
"Rev": "c85f1217d51339c0fa3a498cc8b2075de695dae6"
}
]
}

26
NOTES Normal file
View file

@ -0,0 +1,26 @@
DB Profile
┌───────────▼────────────────────────────┐
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
Bitmap──▶0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
F ▶│0000000000000000000000000000000000000000│
r ││0000000000000000000000000000000000000000│
a ││0000000000000000000000000000000000000000│
m ││0000000000000000000000000000000000000000│
e ▶│0000000000000000000000000000000000000000│
└────────────────────────────────────────┘
▲───────────▲
Slice
Fragment=intersection of frame & slice

22
README.md
View file

@ -24,28 +24,12 @@ Now you can install the `pilosa` binary:
$ go install github.com/umbel/pilosa/...
```
Pilosa requires that [etcd][] is running locally for coordinating the cluster:
```sh
# In another terminal window
$ etcd
```
Now run `pilosa` with the default configuration:
```sh
pilosa
```
This setup assumes that cassandra and etcd are running locally and that
cassandra keyspace pilosa has been setup as describe in the file
`index/storage_cass.go`.
If you don't want backend storage you can comment out storage_backend in the
config file and it will just operate out of memory.
[etcd]: https://github.com/coreos/etcd
## Development
@ -68,9 +52,3 @@ $ godep save ./...
[godep]: https://github.com/tools/godep
### Running tests
Because of a bug in Go 1.5, you'll need to run `make test` to exclude tests
the the `vendor/` directory.

8796
benchmark_autogen_test.go
View file

File diff suppressed because it is too large Load diff

267
bitmap.go
View file

@ -8,8 +8,11 @@ import (
"encoding/base64"
"encoding/binary"
"encoding/gob"
"encoding/json"
"io"
log "github.com/cihub/seelog"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa/internal"
"github.com/yasushi-saito/rbtree"
)
@ -43,6 +46,15 @@ func (b *Bitmap) Chunk(c *Chunk) *Chunk {
// AddChunk adds c to the bitmap.
func (b *Bitmap) AddChunk(c *Chunk) { b.tree.Insert(c) }
// Chunks returns a list of all chunks.
func (b *Bitmap) Chunks() []*Chunk {
var a []*Chunk
for itr := b.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
a = append(a, itr.Item().Clone())
}
return a
}
// ChunkIterator returns an iterator for looping over the bitmap's chunks.
func (b *Bitmap) ChunkIterator() *ChunkIterator {
return &ChunkIterator{b.tree.Min()}
@ -58,18 +70,21 @@ func (b *Bitmap) Clone() *Bitmap {
break
}
node := itr.Item()
other.AddChunk(&Chunk{
Key: node.Key,
Value: node.Value.copy(),
})
other.AddChunk(itr.Item().Clone())
itr = itr.Next()
}
return other
}
// IntersectionCount returns the number of itersections between b and other.
// Merge adds chunks from other to b.
// Chunks in b are overwritten if they exist in other.
func (b *Bitmap) Merge(other *Bitmap) {
for itr := other.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
b.AddChunk(itr.Item().Clone())
}
}
// IntersectionCount returns the number of intersections between b and other.
func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
itr0 := b.ChunkIterator()
itr1 := other.ChunkIterator()
@ -91,8 +106,8 @@ func (b *Bitmap) IntersectionCount(other *Bitmap) uint64 {
return results
}
// Intersection returns the itersection of b and other.
func (b *Bitmap) Intersection(other *Bitmap) *Bitmap {
// Intersect returns the itersection of b and other.
func (b *Bitmap) Intersect(other *Bitmap) *Bitmap {
itr0 := b.ChunkIterator()
itr1 := other.ChunkIterator()
@ -107,7 +122,7 @@ func (b *Bitmap) Intersection(other *Bitmap) *Bitmap {
} else if itr0.Item().Key == itr1.Item().Key {
output.AddChunk(&Chunk{
Key: itr0.Item().Key,
Value: itr0.Item().Value.intersection(itr1.Item().Value),
Value: itr0.Item().Value.intersect(itr1.Item().Value),
})
itr0 = itr0.Next()
itr1 = itr1.Next()
@ -202,7 +217,7 @@ func (b *Bitmap) Difference(other *Bitmap) *Bitmap {
Value: itr0.Item().Value.difference(itr1.Item().Value),
}
// Could not add if all zero
// Do not add if all bits are zeroed.
if chunk.Value.bitcount() > 0 {
output.AddChunk(chunk)
}
@ -236,35 +251,87 @@ func (b *Bitmap) ToRawCompressString() (string, int) {
return base64.StdEncoding.EncodeToString(bt.Bytes()), max_slice
}
// ToBytes returns a gob-encoded byte slice of b.
func (b *Bitmap) ToBytes() []byte {
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
enc.Encode(b.tree.Len())
// WriteTo writes the encoded bitmap to w.
func (b *Bitmap) WriteTo(w io.Writer) (n int64, err error) {
// Wrap output in gzip compression.
z := gzip.NewWriter(w)
// Encode chunk count.
enc := gob.NewEncoder(w)
if err := enc.Encode(b.tree.Len()); err != nil {
return 0, err
}
// Encode all chunks.
for i := b.tree.Min(); !i.Limit(); i = i.Next() {
obj := i.Item().(*Chunk)
err := enc.Encode(obj)
if err != nil {
log.Warn(err)
if err := enc.Encode(i.Item().(*Chunk)); err != nil {
return 0, err
}
}
return buf.Bytes()
// Flush and close.
if err := z.Close(); err != nil {
return 0, err
}
return 0, nil
}
// FromBytes decodes a gob-encoded byte slice into b.
func (b *Bitmap) FromBytes(raw []byte) {
buf := bytes.NewBuffer(raw)
dec := gob.NewDecoder(buf)
// ReadFrom reads encoded bitmap data from r into b.
func (b *Bitmap) ReadFrom(r io.Reader) (n int64, err error) {
// Uncompress from gzip format.
z, err := gzip.NewReader(r)
if err != nil {
return 0, err
}
dec := gob.NewDecoder(z)
// Read size from data.
var size int
dec.Decode(&size)
if err := dec.Decode(&size); err != nil {
return 0, err
}
// Read chunks into bitmap.
b.tree = rbtree.NewTree(rbtreeItemCompare)
for i := 0; i < size; i++ {
var chunk Chunk
dec.Decode(&chunk)
if err := dec.Decode(&chunk); err != nil {
return 0, err
}
b.AddChunk(&chunk)
}
b.SetCount(b.BitCount())
return 0, nil
}
// MarshalJSON returns a JSON-encoded byte slice of b.
func (b *Bitmap) MarshalJSON() ([]byte, error) {
o := bitmapJSON{
Chunks: make([]chunkJSON, 0, b.tree.Len()),
}
for itr := b.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
o.Chunks = append(o.Chunks, chunkJSON{Key: itr.Item().Key, Value: itr.Item().Value})
}
return json.Marshal(&o)
}
// MarshalBinary returns a gob-encoded byte slice of b.
func (b *Bitmap) MarshalBinary() ([]byte, error) {
var buf bytes.Buffer
if _, err := b.WriteTo(&buf); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
// UnmarshalBinary decodes a gob-encoded byte slice into b.
func (b *Bitmap) UnmarshalBinary(data []byte) error {
_, err := b.ReadFrom(bytes.NewReader(data))
return err
}
// Bits returns the bits in b as a slice of ints.
@ -291,7 +358,7 @@ func (b *Bitmap) Bits() []uint64 {
}
// SetBit sets the i-th bit of the bitmap.
func (b *Bitmap) SetBit(i uint64) (bool, *Chunk, Address) {
func (b *Bitmap) SetBit(i uint64) (changed bool) {
address := deref(i)
chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)})
@ -300,29 +367,29 @@ func (b *Bitmap) SetBit(i uint64) (bool, *Chunk, Address) {
b.AddChunk(chunk)
}
changed := chunk.Value.setBit(address.BlockIndex, address.Bit)
changed = chunk.Value.setBit(address.BlockIndex, address.Bit)
if changed {
b.bcount++
}
return changed, chunk, address
return changed
}
// ClearBit clears the i-th bit of the bitmap.
func (b *Bitmap) ClearBit(i uint64) (bool, *Chunk, Address) {
func (b *Bitmap) ClearBit(i uint64) (changed bool) {
address := deref(i)
chunk := b.Chunk(&Chunk{address.ChunkKey, make(Blocks, 32)})
if chunk == nil {
return false, nil, address
return false
}
changed := chunk.Value.clearBit(address.BlockIndex, address.Bit)
changed = chunk.Value.clearBit(address.BlockIndex, address.Bit)
if changed && b.bcount > 0 {
b.bcount--
}
return changed, chunk, address
return changed
}
// Len returns the number of chunks in b.
@ -343,12 +410,75 @@ func (b *Bitmap) BitCount() uint64 {
return n
}
// encodeBitmap converts b into its internal representation.
func encodeBitmap(b *Bitmap) *internal.Bitmap {
pb := &internal.Bitmap{}
for i := b.tree.Min(); !i.Limit(); i = i.Next() {
pb.Chunks = append(pb.Chunks, encodeChunk(i.Item().(*Chunk)))
}
return pb
}
// decodeBitmap converts b from its internal representation.
func decodeBitmap(pb *internal.Bitmap) *Bitmap {
b := NewBitmap()
for _, chunk := range pb.GetChunks() {
b.AddChunk(decodeChunk(chunk))
}
b.SetCount(b.BitCount())
return b
}
// Union performs a union on a slice of bitmaps.
func Union(bitmaps []*Bitmap) *Bitmap {
other := bitmaps[0]
for _, bm := range bitmaps[1:] {
other = other.Union(bm)
}
return other
}
// bitmapJSON is the JSON representation of Bitmap.
type bitmapJSON struct {
Chunks []chunkJSON `json:"chunks"`
}
// Chunk represents a set of blocks in a Bitmap.
type Chunk struct {
Key uint64
Value Blocks
}
// Clone returns a copy of c.
func (c *Chunk) Clone() *Chunk {
return &Chunk{
Key: c.Key,
Value: c.Value.copy(),
}
}
// encodeChunks encodes c into its internal representation.
func encodeChunk(c *Chunk) *internal.Chunk {
return &internal.Chunk{
Key: proto.Uint64(c.Key),
Value: []uint64(c.Value),
}
}
// decodeChunk decodes c from its internal representation.
func decodeChunk(pb *internal.Chunk) *Chunk {
return &Chunk{
Key: pb.GetKey(),
Value: Blocks(pb.GetValue()),
}
}
// chunkJSON is the JSON representation of Chunk.
type chunkJSON struct {
Key uint64
Value []uint64
}
// ChunkIterator represents an object for iterating over chunks in a bitmap.
type ChunkIterator struct {
itr rbtree.Iterator
@ -383,6 +513,73 @@ func rbtreeItemCompare(a, b rbtree.Item) int {
return 0
}
type Blocks []uint64
// NewBlocks returns a 32-length Block.
func NewBlocks() Blocks {
return make(Blocks, 32)
}
func (a Blocks) bitcount() uint64 {
return popcntSlice(a)
}
func (a Blocks) union(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] | other[i]
}
return ret
}
func (a Blocks) invert() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = ^a[i]
}
return other
}
func (a Blocks) copy() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = a[i]
}
return other
}
func (a Blocks) andcount(other Blocks) uint64 {
return popcntAndSliceAsm(a, other)
}
func (a Blocks) intersect(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] & other[i]
}
return ret
}
func (a Blocks) difference(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] &^ other[i]
}
return ret
}
func (a Blocks) setBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] |= 1 << bit
return val == 0
}
func (a Blocks) clearBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] &= ^(1 << bit)
return val != 0
}
// Address represents a location for a given chunk/block/bit.
type Address struct {
ChunkKey uint64

74
blocks.go
View file

@ -1,74 +0,0 @@
package pilosa
import (
"fmt"
)
type Blocks []uint64
// NewBlocks returns a 32-length Block.
func NewBlocks() Blocks {
return make(Blocks, 32)
}
func (a Blocks) bitcount() uint64 {
fmt.Println("bitcount:", a)
return popcntSlice(a)
}
func (a Blocks) union(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] | other[i]
}
return ret
}
func (a Blocks) invert() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = ^a[i]
}
return other
}
func (a Blocks) copy() Blocks {
other := NewBlocks()
for i, _ := range a {
other[i] = a[i]
}
return other
}
func (a Blocks) andcount(other Blocks) uint64 {
println("andcount")
return popcntAndSliceAsm(a, other)
}
func (a Blocks) intersection(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] & other[i]
}
return ret
}
func (a Blocks) difference(other Blocks) Blocks {
ret := NewBlocks()
for i, _ := range a {
ret[i] = a[i] &^ other[i]
}
return ret
}
func (a Blocks) setBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] |= 1 << bit
return val == 0
}
func (a Blocks) clearBit(i uint8, bit uint8) (changed bool) {
val := a[i] & (1 << bit)
a[i] &= ^(1 << bit)
return val != 0
}

448
brand.go
View file

@ -1,448 +0,0 @@
package pilosa
import (
"encoding/json"
"fmt"
"math/rand"
"sort"
"sync"
"time"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa/statsd"
)
var FragmentBase string
var globalLock sync.Mutex
type Pair struct {
Key, Count uint64
}
type Rank struct {
*Pair
bitmap *Bitmap
category uint64
}
type RankList []*Rank
func (p RankList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p RankList) Len() int { return len(p) }
func (p RankList) Less(i, j int) bool { return p[i].Count > p[j].Count }
type Brand struct {
bitmap_cache map[uint64]*Rank
db string
frame string
slice int
storage Storage
rankings RankList
rank_count int
threshold_value uint64
threshold_length int
threshold_idx int
skip int
rank_time time.Time
}
func NewBrand(db string, frame string, slice int, s Storage, threshold_len int, threshold int, skipp int) *Brand {
f := new(Brand)
f.storage = s
f.frame = frame
f.slice = slice
f.db = db
f.rank_count = 0
f.threshold_value = 0
f.threshold_length = threshold_len
f.threshold_idx = threshold
f.skip = skipp
f.Clear() //alloc the cache
return f
}
func (b *Brand) Clear() bool {
b.bitmap_cache = make(map[uint64]*Rank)
return true
}
func (b *Brand) Exists(bitmap_id uint64) bool {
_, ok := b.bitmap_cache[bitmap_id]
return ok
}
func (b *Brand) Get(bitmap_id uint64) *Bitmap {
if bm, ok := b.bitmap_cache[bitmap_id]; ok {
return bm.bitmap
}
// I should fetch the category here..need to come up with a good source
bm, filter := b.storage.Fetch(bitmap_id, b.db, b.frame, b.slice)
b.cache_it(bm, bitmap_id, filter)
return bm
}
func (b *Brand) Get_nocache(bitmap_id uint64) (*Bitmap, uint64) {
if bm, ok := b.bitmap_cache[bitmap_id]; ok {
return bm.bitmap, bm.category
}
//I should fetch the category here..need to come up with a good source
return b.storage.Fetch(bitmap_id, b.db, b.frame, b.slice)
}
func (b *Brand) GetFilter(bitmap_id, filter uint64) *Bitmap {
bm, old_filter := b.storage.Fetch(bitmap_id, b.db, b.frame, b.slice)
if filter == 0 {
filter = old_filter
}
b.cache_it(bm, bitmap_id, filter)
return bm
}
func (b *Brand) cache_it(bm *Bitmap, bitmap_id uint64, category uint64) {
if bm.Count() >= b.threshold_value {
b.bitmap_cache[bitmap_id] = &Rank{&Pair{bitmap_id, bm.Count()}, bm, category}
if len(b.bitmap_cache) > b.threshold_length {
log.Info("RANK:", len(b.bitmap_cache), b.threshold_length, b.threshold_value)
b.Rank()
b.trim()
}
}
}
func (b *Brand) trim() {
for k, item := range b.bitmap_cache {
if item.bitmap.Count() <= b.threshold_value {
delete(b.bitmap_cache, k)
}
}
log.Info("TRIM:", len(b.bitmap_cache), b.threshold_length)
}
func (b *Brand) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool {
bm1, ok := b.bitmap_cache[bitmap_id]
var bm *Bitmap
if ok {
bm = bm1.bitmap
} else {
bm = b.GetFilter(bitmap_id, filter) //aways overwrites what is in cass filter type
}
change, chunk, address := bm.SetBit(bit_pos)
if change {
val := chunk.Value[address.BlockIndex]
b.storage.StoreBit(bitmap_id, b.db, b.frame, b.slice, filter, address.ChunkKey, int32(address.BlockIndex), val, bm.Count())
b.rank_count++
}
return change
}
func (b *Brand) Rank() {
start := time.Now()
var list RankList
for k, item := range b.bitmap_cache {
list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap, item.category})
}
sort.Sort(list)
b.rankings = list
if len(list) > b.threshold_idx {
item := list[b.threshold_idx]
b.threshold_value = item.bitmap.Count()
} else {
b.threshold_value = 1
}
b.rank_count = 0
delta := time.Since(start)
statsd.SendTimer("brand_Rank", delta.Nanoseconds())
b.rank_time = start
}
func packagePairs(r RankList) []Pair {
res := make([]Pair, r.Len())
for i, v := range r {
res[i] = Pair{v.Key, v.Count}
}
return res
}
func (b *Brand) Stats() interface{} {
total := uint64(0)
i := uint64(0)
bit_total := uint64(0)
for _, v := range b.bitmap_cache {
total += uint64(v.bitmap.Len()) * uint64(256)
i += 1
bit_total += v.Count
}
avg_bytes := uint64(0)
avg_bits := uint64(0)
if i > 0 {
avg_bytes = total / i
avg_bits = bit_total / i
}
stats := map[string]interface{}{
"total size of cache in bytes": total,
"number of bitmaps": len(b.bitmap_cache),
"avg size of bitmap in space(bytes)": avg_bytes,
"avg size of bitmap in bits": avg_bits,
"rank counter": b.rank_count,
"threshold_value": b.threshold_value,
"threshold_length": b.threshold_length,
"threshold_idx": b.threshold_idx,
"skip": b.skip}
return stats
}
func (b *Brand) Store(bitmap_id uint64, bm *Bitmap, filter uint64) error {
if err := b.storage.Store(bitmap_id, b.db, b.frame, b.slice, filter, bm); err != nil {
return err
}
b.cache_it(bm, bitmap_id, filter)
return nil
}
func (b *Brand) checkRank() {
if len(b.rankings) < 50 {
b.Rank()
return
}
if b.rank_count > 0 {
last := time.Since(b.rank_time) * time.Second
if last > 60*5 {
b.Rank()
}
}
}
func (b *Brand) TopN(src_bitmap *Bitmap, n int, categories []uint64) []Pair {
b.checkRank()
set := make(map[uint64]struct{})
for _, v := range categories {
set[v] = struct{}{}
}
return b.TopNCat(src_bitmap, n, set)
}
func dump(r RankList, n int) {
for i, v := range r {
log.Info(i, v)
if i > n {
return
}
}
}
func (b *Brand) TopNAll(n int, categories []uint64) []Pair {
log.Trace("TopNAll")
b.checkRank()
results := make([]Pair, 0, 0)
set := make(map[uint64]struct{})
needCat := false
for _, v := range categories {
set[v] = struct{}{}
needCat = true
}
count := 0
for _, pair := range b.rankings {
if needCat {
if _, ok := set[pair.category]; !ok {
continue
}
}
if count >= n {
break
}
if pair.Count > 0 {
results = append(results, Pair{pair.Key, pair.Count})
}
count++
}
return results
}
func (b *Brand) TopNCat(src_bitmap *Bitmap, n int, set map[uint64]struct{}) []Pair {
breakout := 1000
var (
o *Rank
results RankList
)
counter := 0
x := 0
needCat := (len(set) > 0)
for i, pair := range b.rankings {
if needCat {
if _, ok := set[pair.category]; !ok {
continue
}
}
if counter > n {
break
}
bc := src_bitmap.IntersectionCount(pair.bitmap)
if bc > 0 {
results = append(results, &Rank{&Pair{pair.Key, bc}, nil, pair.category})
counter = counter + 1
}
x = i
}
sort.Sort(results)
if counter < n {
return packagePairs(results)
}
end := len(results) - 1
o = results[end]
current_threshold := o.Count
if current_threshold <= 10 {
return packagePairs(results)
}
results = append(results, o)
for i := x + 1; i < len(b.rankings); i++ {
o = b.rankings[i]
if needCat {
if _, ok := set[o.category]; !ok {
continue
}
counter = counter + 1
} else {
counter = counter + 1
}
if counter > breakout {
break
}
//if o.Count < current_threshold { //done
//need something to do with the size of initianl bitmap
if o.Count < current_threshold { //done
break
}
bc := src_bitmap.IntersectionCount(o.bitmap)
if bc > current_threshold {
if results[end-1].Count > bc {
results[end] = &Rank{&Pair{o.Key, bc}, nil, o.category}
current_threshold = bc
} else {
results[end+1] = &Rank{&Pair{o.Key, bc}, nil, o.category}
sort.Sort(results)
o = results[end]
current_threshold = o.Count
}
}
}
return packagePairs(results[:end])
}
func (b *Brand) getFileName() string {
base := FragmentBase
if base == "" {
base = "."
}
return fmt.Sprintf("%s/%s.%s.%d.json", base, b.db, b.frame, b.slice)
}
func (b *Brand) Persist() error {
log.Info("Brand Persist:", b.getFileName())
b.storage.Flush()
asize := len(b.bitmap_cache)
if asize == 0 {
log.Warn("Nothing to save ", b.getFileName())
return nil
}
w, err := createFile(b.getFileName())
if err != nil {
log.Warn("Error opening outfile ", b.getFileName())
log.Warn(err)
return err
}
defer w.Close()
defer b.storage.Close()
var list RankList
for k, item := range b.bitmap_cache {
list = append(list, &Rank{&Pair{k, item.bitmap.Count()}, item.bitmap, item.category})
}
sort.Sort(list)
results := make([]uint64, asize)
i := 0
for _, k := range list { // map[uint64]*Rank
results[i] = k.Key
i += 1
}
encoder := json.NewEncoder(w)
return encoder.Encode(results)
}
func (b *Brand) Load(f *Fragment) {
log.Warn("Brand Load")
time.Sleep(time.Duration(rand.Intn(32)) * time.Second) //trying to avoid mass cassandra hit
r, err := openFile(b.getFileName())
if err != nil {
log.Warn("NO Brand Init File:", b.getFileName())
return
}
dec := json.NewDecoder(r)
var keys []uint64
if err := dec.Decode(&keys); err != nil {
return
}
globalLock.Lock()
defer globalLock.Unlock()
// probaly need to get a etcd lock too someday
for _, k := range keys {
b.Get(k)
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond) //trying to avoid mass cassandra hit
}
}
func (b *Brand) ClearBit(bitmap_id uint64, bit_pos uint64) bool {
log.Trace("ClearBit", bitmap_id, bit_pos)
bm1, ok := b.bitmap_cache[bitmap_id]
var bm *Bitmap
filter := uint64(0)
if ok {
bm = bm1.bitmap
filter = bm1.category
} else {
bm, filter = b.Get_nocache(bitmap_id)
if bm.Count() == 0 {
return false //nothing to unset
}
}
changed, chunk, address := bm.ClearBit(bit_pos)
if changed {
val := chunk.Value[address.BlockIndex]
if val == 0 {
b.storage.RemoveBit(bitmap_id, b.db, b.frame, b.slice, filter, address.ChunkKey, int32(address.BlockIndex), bm.Count())
} else {
b.storage.StoreBit(bitmap_id, b.db, b.frame, b.slice, filter, address.ChunkKey, int32(address.BlockIndex), val, bm.Count())
}
b.rank_count++
}
return changed
}

44
brand_test.go
View file

@ -1,44 +0,0 @@
package pilosa_test
import (
"math/rand"
"testing"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
"github.com/umbel/pilosa/storage/mem"
)
var (
size int
membrand *pilosa.Brand
cassbrand *pilosa.Brand
)
func init() {
size = 1000
statsd.Setup()
// SetupCassandra()
membrand = pilosa.NewBrand("db", "frame", 0, mem.NewStorage(), size, size, 0)
for i := uint64(0); i < uint64(size); i++ {
membrand.SetBit(i, 0, 1)
}
// cassbrand = NewBrand("db", "frame", 0, NewCassStorage(), size, size, 0)
// for i := uint64(0); i < uint64(size); i++ {
// cassbrand.SetBit(i, 0, 1)
// }
}
func benchmarkBrand(b *testing.B, size int, fill int, brand *pilosa.Brand) {
println(b.N)
for i := 0; i < b.N; i++ {
bid := rand.Int() % size
profile := uint64(i % fill)
brand.SetBit(uint64(bid), profile, 1)
}
}
func BenchmarkBrandMemSetBitL2(b *testing.B) { benchmarkBrand(b, size, 1024*64, membrand) }
func BenchmarkBrandCasSetBitL2(b *testing.B) { benchmarkBrand(b, size, 1024*64, cassbrand) }

262
cache.go Normal file
View file

@ -0,0 +1,262 @@
package pilosa
import (
"encoding/json"
"io"
"sort"
"time"
"github.com/gogo/protobuf/proto"
"github.com/golang/groupcache/lru"
"github.com/umbel/pilosa/internal"
)
// Cache represents a cache for bitmaps.
type Cache interface {
io.WriterTo
io.ReaderFrom
Add(bitmapID, category uint64, bm *Bitmap)
Get(bitmapID uint64) (bm *Bitmap, ok bool)
Len() int
// Updates the cache, if necessary.
Invalidate()
// Returns a list of all key/count pairs.
Pairs() []Pair
}
// LRUCache represents a least recently used Cache implemenation.
type LRUCache struct {
cache *lru.Cache
keys map[uint64]struct{}
}
// NewLRUCache returns a new instance of LRUCache.
func NewLRUCache(maxEntries int) *LRUCache {
c := &LRUCache{
cache: lru.New(maxEntries),
keys: make(map[uint64]struct{}),
}
c.cache.OnEvicted = c.onEvicted
return c
}
// Get returns a bitmap with a given id.
func (c *LRUCache) Add(bitmapID, category uint64, bm *Bitmap) {
c.cache.Add(bitmapID, bm)
c.keys[bitmapID] = struct{}{}
}
// Get returns a bitmap with a given id.
func (c *LRUCache) Get(bitmapID uint64) (bm *Bitmap, ok bool) {
value, ok := c.cache.Get(bitmapID)
if !ok {
return nil, false
}
return value.(*Bitmap), true
}
// Len returns the number of items in the cache.
func (c *LRUCache) Len() int { return c.cache.Len() }
// Invalidate is a no-op.
func (c *LRUCache) Invalidate() {}
// Pairs returns all key/count pairs in the cache.
func (c *LRUCache) Pairs() []Pair {
a := make([]Pair, 0, len(c.keys))
for k := range c.keys {
bm, _ := c.Get(k)
a = append(a, Pair{
Key: k,
Count: bm.Count(),
})
}
return a
}
// WriteTo writes the cache to w.
func (c *LRUCache) WriteTo(w io.Writer) (n int64, err error) {
// Write keys to slice.
a := make([]uint64, 0, len(c.keys))
for k := range c.keys {
a = append(a, k)
}
// Encode to file as array of keys.
if err := json.NewEncoder(w).Encode(a); err != nil {
return 0, err
}
return 0, nil
}
// ReadFrom read from r into the cache.
func (c *LRUCache) ReadFrom(r io.Reader) (n int64, err error) {
var keys []uint64
if err := json.NewDecoder(r).Decode(&keys); err != nil {
return 0, err
}
panic("FIXME: TODO")
}
func (c *LRUCache) onEvicted(key lru.Key, _ interface{}) {
delete(c.keys, key.(uint64))
}
// Ensure LRUCache implements Cache.
var _ Cache = &LRUCache{}
// RankCache represents a cache with sorted entries.
type RankCache struct {
entries map[uint64]*Pair
rankings []Pair // cached, ordered list
updateN int
updateTime time.Time
ThresholdLength int
ThresholdIndex int
ThresholdValue uint64
}
// NewRankCache returns a new instance of RankCache.
func NewRankCache() *RankCache {
return &RankCache{
entries: make(map[uint64]*Pair),
}
}
// Get returns a bitmap with a given id.
func (c *RankCache) Add(bitmapID, category uint64, bm *Bitmap) {
// Ignore if the bit count on the bitmap is below the threshold.
if bm.Count() < c.ThresholdValue {
return
}
// Add to cache.
c.entries[bitmapID] = &Pair{
Key: bitmapID,
Count: bm.Count(),
bitmap: bm,
category: category,
}
// If size is larger than the threshold then trim it.
if len(c.entries) > c.ThresholdLength {
c.update()
for k, entry := range c.entries {
if entry.bitmap.Count() <= c.ThresholdValue {
delete(c.entries, k)
}
}
}
}
// Get returns a bitmap with a given id.
func (c *RankCache) Get(bitmapID uint64) (bm *Bitmap, ok bool) {
entry, ok := c.entries[bitmapID]
if !ok {
return nil, false
}
return entry.bitmap, true
}
// Len returns the number of items in the cache.
func (c *RankCache) Len() int { return len(c.entries) }
// Invalidate reorders the entries, if necessary.
func (c *RankCache) Invalidate() {
// Update if there aren't many items or it hasn't been updated recently.
if len(c.rankings) < 50 || (c.updateN > 0 && time.Since(c.updateTime) > 5*time.Minute) {
c.update()
}
}
// update reorders the entries by rank.
func (c *RankCache) update() {
// Convert cache to a sorted list.
list := make([]Pair, 0, len(c.entries))
for k, item := range c.entries {
list = append(list, Pair{
Key: k,
Count: item.bitmap.Count(),
bitmap: item.bitmap,
category: item.category,
})
}
sort.Sort(Pairs(list))
// Store the count of the item at the threshold index.
c.rankings = list
if len(c.rankings) > c.ThresholdIndex {
c.ThresholdValue = list[c.ThresholdIndex].bitmap.Count()
} else {
c.ThresholdValue = 1
}
// Reset counters.
c.updateTime, c.updateN = time.Now(), 0
}
// Pairs returns an ordered list of key/count pairs.
func (c *RankCache) Pairs() []Pair { return c.rankings }
// WriteTo writes the cache to w.
func (c *RankCache) WriteTo(w io.Writer) (n int64, err error) {
panic("FIXME: TODO")
}
// ReadFrom read from r into the cache.
func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
panic("FIXME: TODO")
}
// Ensure RankCache implements Cache.
var _ Cache = &RankCache{}
type Pair struct {
Key uint64 `json:"key"`
Count uint64 `json:"count"`
bitmap *Bitmap
category uint64
}
func encodePair(p Pair) *internal.Pair {
return &internal.Pair{
Key: proto.Uint64(p.Key),
Count: proto.Uint64(p.Count),
}
}
func decodePair(pb *internal.Pair) Pair {
return Pair{
Key: pb.GetKey(),
Count: pb.GetCount(),
}
}
type Pairs []Pair
func (p Pairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p Pairs) Len() int { return len(p) }
func (p Pairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
func encodePairs(a Pairs) []*internal.Pair {
other := make([]*internal.Pair, len(a))
for i := range a {
other[i] = encodePair(a[i])
}
return other
}
func decodePairs(a []*internal.Pair) []Pair {
other := make([]Pair, len(a))
for i := range a {
other[i] = decodePair(a[i])
}
return other
}

123
cluster.go Normal file
View file

@ -0,0 +1,123 @@
package pilosa
import (
"encoding/binary"
"hash/fnv"
)
const (
// DefaultPartitionN is the default number of partitions in a cluster.
DefaultPartitionN = 64
// DefaultReplicaN is the default number of replicas per partition.
DefaultReplicaN = 1
)
// Node represents a node in the cluster.
type Node struct {
Host string
}
// Nodes represents a list of nodes.
type Nodes []*Node
// ContainsHost returns true if host matches on of the node's host.
func (a Nodes) ContainsHost(host string) bool {
for _, n := range a {
if n.Host == host {
return true
}
}
return false
}
// Hosts returns a list of all hostnames.
func (a Nodes) Hosts() []string {
hosts := make([]string, len(a))
for i, n := range a {
hosts[i] = n.Host
}
return hosts
}
// Cluster represents a collection of nodes.
type Cluster struct {
Nodes []*Node
// Hashing algorithm used to assign partitions to nodes.
Hasher Hasher
// The number of partitions in the cluster.
PartitionN int
// The number of replicas a partition has.
ReplicaN int
}
// NewCluster returns a new instance of Cluster with defaults.
func NewCluster() *Cluster {
return &Cluster{
Hasher: &jmphasher{},
PartitionN: DefaultPartitionN,
ReplicaN: DefaultReplicaN,
}
}
// Partition returns the partition that a slice belongs to.
func (c *Cluster) Partition(slice uint64) int {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], slice)
// Hash the bytes and mod by partition count.
h := fnv.New64a()
h.Write(buf[:])
return int(h.Sum64() % uint64(c.PartitionN))
}
// SliceNodes returns a list of nodes that own a slice.
func (c *Cluster) SliceNodes(slice uint64) []*Node {
return c.PartitionNodes(c.Partition(slice))
}
// PartitionNodes returns a list of nodes that own a partition.
func (c *Cluster) PartitionNodes(partitionID int) []*Node {
// Default replica count to between one and the number of nodes.
// The replica count can be zero if there are no nodes.
replicaN := c.ReplicaN
if replicaN > len(c.Nodes) {
replicaN = len(c.Nodes)
} else if replicaN == 0 {
replicaN = 1
}
// Determine primary owner node.
index := c.Hasher.Hash(uint64(partitionID), len(c.Nodes))
// Collect nodes around the ring.
nodes := make([]*Node, replicaN)
for i := 0; i < replicaN; i++ {
nodes[i] = c.Nodes[(index+i)%len(c.Nodes)]
}
return nodes
}
// Hasher represents an interface to hash integers into buckets.
type Hasher interface {
// Hashes the key into a number between [0,N).
Hash(key uint64, n int) int
}
// jmphasher represents an implementation of jmphash. Implements Hasher.
type jmphasher struct{}
// Hash returns the integer hash for the given key.
func (h *jmphasher) Hash(key uint64, n int) int {
b, j := int64(-1), int64(0)
for j < int64(n) {
b = j
key = key*uint64(2862933555777941757) + 1
j = int64(float64(b+1) * (float64(int64(1)<<31) / float64((key>>33)+1)))
}
return int(b)
}

90
cluster_test.go Normal file
View file

@ -0,0 +1,90 @@
package pilosa_test
import (
"fmt"
"math/rand"
"reflect"
"testing"
"testing/quick"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
)
// Ensure the cluster can fairly distribute partitions across the nodes.
func TestCluster_Owners(t *testing.T) {
c := pilosa.Cluster{
Nodes: []*pilosa.Node{
{Host: "serverA:1000"},
{Host: "serverB:1000"},
{Host: "serverC:1000"},
},
Hasher: NewModHasher(),
ReplicaN: 2,
}
// Verify nodes are distributed.
if a := c.PartitionNodes(0); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[0], c.Nodes[1]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
// Verify nodes go around the ring.
if a := c.PartitionNodes(2); !reflect.DeepEqual(a, []*pilosa.Node{c.Nodes[2], c.Nodes[0]}) {
t.Fatalf("unexpected owners: %s", spew.Sdump(a))
}
}
// Ensure the partitioner can assign a fragment to a partition.
func TestCluster_Partition(t *testing.T) {
if err := quick.Check(func(slice uint64, partitionN int) bool {
c := pilosa.NewCluster()
c.PartitionN = partitionN
partitionID := c.Partition(slice)
if partitionID < 0 || partitionID > partitionN {
t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN)
}
return true
}, &quick.Config{
Values: func(values []reflect.Value, rand *rand.Rand) {
values[0] = reflect.ValueOf(uint64(rand.Uint32()))
values[1] = reflect.ValueOf(rand.Intn(1000) + 1)
},
}); err != nil {
t.Fatal(err)
}
}
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewCluster(n int) *pilosa.Cluster {
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
Host: fmt.Sprintf("host%d", i),
})
}
return c
}
// ModHasher represents a simple, mod-based hashing.
type ModHasher struct{}
// NewModHasher returns a new instance of ModHasher with n buckets.
func NewModHasher() *ModHasher { return &ModHasher{} }
func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
// ConstHasher represents hash that always returns the same index.
type ConstHasher struct {
i int
}
// NewConstHasher returns a new instance of ConstHasher that always returns i.
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }

111
cmd/pilosa/config.go
View file

@ -4,106 +4,55 @@ import (
"time"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
_ "github.com/umbel/pilosa/storage"
"github.com/umbel/pilosa/storage/cassandra"
"github.com/umbel/pilosa/transport"
)
const (
// DefaultLogPath is the default file path where the log will be written.
DefaultLogPath = "/tmps"
// DefaultHost is the default hostname to use.
DefaultHost = "localhost"
// DefaultLogLevel is the default logging level used by seelog.
DefaultLogLevel = "info"
// DefaultFragmentBase is the default path where fragments are stored.
DefaultFragmentBase = "/tmp/single"
)
var (
// DefaultSupportedFrames are the frames supported by default.
DefaultSupportedFrames = [...]string{"b.n", "t.t", "l.n", "d", "p.n"}
// DefaultETCDHosts are the default hosts to connect to.
DefaultETCDHosts = [...]string{"http://127.0.0.1:4001"}
// DefaultAddr is the default HTTP address to use.
DefaultAddr = ":15000"
)
// Config represents the configuration for the command.
type Config struct {
ID *pilosa.GUID `toml:"id"`
Host string `toml:"host"`
Host string `toml:"host"`
Addr string `toml:"addr"`
TCP struct {
Port int `toml:"port"`
} `toml:"tcp"`
HTTP struct {
Port int `toml:"port"`
DefaultDB string `toml:"default-db"`
RequestLogPath string `toml:"request-log-path"`
SetBitLogEnabled bool `toml:"set-bit-log-enabled"`
} `toml:"http"`
Log struct {
Path string `toml:"path"`
Level string `toml:"level"`
}
Storage struct {
Backend string `toml:"backend"`
Hosts []string `toml:"hosts"`
Keyspace string `toml:"keyspace"`
FragmentBase string `toml:"fragment-base"`
SupportedFrames []string `toml:"supported-frames"`
CassandraTimeWindow Duration `toml:"cassandra-time-window"`
CassandraMaxSizeBatch int `toml:"cassandra-max-size-batch"`
} `toml:"storage"`
AWS struct {
AccessKeyID string `toml:"access-key-id"`
SecretAccessKey string `toml:"secret-access-key"`
} `toml:"aws"`
LevelDB struct {
Path string `toml:"path"`
} `toml:"leveldb"`
Statsd struct {
Host string `toml:"host"`
} `toml:"statsd"`
Cluster struct {
ReplicaN int `toml:"replicas"`
Nodes []struct {
Host string `toml:"host"`
} `toml:"nodes"`
} `toml:"cluster"`
Plugins struct {
Path string `toml:"path"`
} `toml:"plugins"`
ETCD struct {
Hosts []string `toml:"hosts"`
FragmentAllocLockTTL Duration `toml:"fragment-alloc-lock-ttl"`
} `toml:"etcd"`
}
// NewConfig returns an instance of Config with default options.
func NewConfig() Config {
var c Config
c.Host = "localhost"
c.TCP.Port = transport.DefaultTCPPort
c.HTTP.Port = transport.DefaultHTTPPort
c.Log.Path = DefaultLogPath
c.Storage.Backend = pilosa.DefaultBackend
c.Storage.Hosts = cassandra.DefaultHosts
c.Storage.Keyspace = cassandra.DefaultKeyspace
c.Storage.FragmentBase = DefaultFragmentBase
c.Storage.SupportedFrames = DefaultSupportedFrames[:]
c.Storage.CassandraTimeWindow = Duration(cassandra.DefaultFlushInterval)
c.Storage.CassandraMaxSizeBatch = cassandra.DefaultFlushThreshold
c.Statsd.Host = statsd.DefaultHost
c.ETCD.Hosts = DefaultETCDHosts[:]
func NewConfig() *Config {
c := &Config{
Host: DefaultHost,
Addr: DefaultAddr,
}
c.Cluster.ReplicaN = pilosa.DefaultReplicaN
return c
}
// PilosaCluster returns a new instance of pilosa.Cluster based on the config.
func (c *Config) PilosaCluster() *pilosa.Cluster {
cluster := pilosa.NewCluster()
cluster.ReplicaN = c.Cluster.ReplicaN
for _, n := range c.Cluster.Nodes {
cluster.Nodes = append(cluster.Nodes, &pilosa.Node{Host: n.Host})
}
return cluster
}
// Duration is a TOML wrapper type for time.Duration.
type Duration time.Duration

159
cmd/pilosa/config_test.go
View file

@ -1,157 +1,27 @@
package main_test
import (
"reflect"
"testing"
"time"
"github.com/BurntSushi/toml"
"github.com/umbel/pilosa/cmd/pilosa"
)
// Ensure the ID can be parsed as a GUID.
func TestConfig_Parse_ID(t *testing.T) {
if c, err := ParseConfig(`
id = "00000000-0000-0000-0000-000000000001"
`); err != nil {
t.Fatal(err)
} else if c.ID == nil || c.ID.String() != `00000000-0000-0000-0000-000000000001` {
t.Fatalf("unexpected id: %s", c.ID)
}
}
// Ensure that parsing an invalid GUID returns an error.
func TestConfig_Parse_ID_ErrInvalid(t *testing.T) {
if _, err := ParseConfig(`
id = "x"
`); err == nil || err.Error() != `Type mismatch for 'main.Config.id': invalid GUID "x"` {
t.Fatal(err)
}
}
// Ensure the host can be parsed.
func TestConfig_Parse_Host(t *testing.T) {
if c, err := ParseConfig(`host = "localhost"`); err != nil {
if c, err := ParseConfig(`host = "local"`); err != nil {
t.Fatal(err)
} else if c.Host != "localhost" {
} else if c.Host != "local" {
t.Fatalf("unexpected host: %s", c.Host)
}
}
// Ensure the "tcp" config can be parsed.
func TestConfig_Parse_TCP(t *testing.T) {
if c, err := ParseConfig(`
[tcp]
port = 123
`); err != nil {
// Ensure the addr can be parsed.
func TestConfig_Parse_Addr(t *testing.T) {
if c, err := ParseConfig(`addr = ":80"`); err != nil {
t.Fatal(err)
} else if c.TCP.Port != 123 {
t.Fatalf("unexpected port: %s", c.TCP.Port)
}
}
// Ensure the "http" config can be parsed.
func TestConfig_Parse_HTTP(t *testing.T) {
if c, err := ParseConfig(`
[http]
port = 123
default-db = "xyz"
request-log-path = "/path/to/log"
set-bit-log-enabled = true
`); err != nil {
t.Fatal(err)
} else if c.HTTP.Port != 123 {
t.Fatalf("unexpected port: %s", c.HTTP.Port)
} else if c.HTTP.DefaultDB != "xyz" {
t.Fatalf("unexpected default db: %s", c.HTTP.DefaultDB)
} else if c.HTTP.RequestLogPath != "/path/to/log" {
t.Fatalf("unexpected request log path: %s", c.HTTP.RequestLogPath)
} else if c.HTTP.SetBitLogEnabled != true {
t.Fatalf("unexpected set bit log enabled: %v", c.HTTP.SetBitLogEnabled)
}
}
// Ensure the "log" config can be parsed.
func TestConfig_Parse_Log(t *testing.T) {
if c, err := ParseConfig(`
[log]
path = "/path/to/log"
level = "debug"
`); err != nil {
t.Fatal(err)
} else if c.Log.Path != "/path/to/log" {
t.Fatalf("unexpected path: %s", c.Log.Path)
} else if c.Log.Level != "debug" {
t.Fatalf("unexpected level: %s", c.Log.Level)
}
}
// Ensure the "storage" config can be parsed.
func TestConfig_Parse_Storage(t *testing.T) {
if c, err := ParseConfig(`
[storage]
backend = "cassandra"
hosts = ["server0", "server1"]
keyspace = "pilosa"
fragment-base = "/path/to/base"
supported-frames = ["a", "b", "c"]
cassandra-time-window = "5s"
cassandra-max-size-batch = 50
`); err != nil {
t.Fatal(err)
} else if c.Storage.Backend != "cassandra" {
t.Fatalf("unexpected backend: %s", c.Storage.Backend)
} else if !reflect.DeepEqual(c.Storage.Hosts, []string{"server0", "server1"}) {
t.Fatalf("unexpected hosts: %+v", c.Storage.Hosts)
} else if c.Storage.Keyspace != "pilosa" {
t.Fatalf("unexpected keyspace: %s", c.Storage.Keyspace)
} else if c.Storage.FragmentBase != "/path/to/base" {
t.Fatalf("unexpected fragment base: %s", c.Storage.FragmentBase)
} else if !reflect.DeepEqual(c.Storage.SupportedFrames, []string{"a", "b", "c"}) {
t.Fatalf("unexpected supported frames: %s", c.Storage.SupportedFrames)
} else if time.Duration(c.Storage.CassandraTimeWindow) != 5*time.Second {
t.Fatalf("unexpected cassandra time window: %s", time.Duration(c.Storage.CassandraTimeWindow))
} else if c.Storage.CassandraMaxSizeBatch != 50 {
t.Fatalf("unexpected cassandra max size batch: %s", c.Storage.CassandraMaxSizeBatch)
}
}
// Ensure the "aws" config can be parsed.
func TestConfig_Parse_AWS(t *testing.T) {
if c, err := ParseConfig(`
[aws]
access-key-id = "abc"
secret-access-key = "def"
`); err != nil {
t.Fatal(err)
} else if c.AWS.AccessKeyID != "abc" {
t.Fatalf("unexpected access key id: %s", c.AWS.AccessKeyID)
} else if c.AWS.SecretAccessKey != "def" {
t.Fatalf("unexpected secret access key: %s", c.AWS.SecretAccessKey)
}
}
// Ensure the "leveldb" config can be parsed.
func TestConfig_Parse_LevelDB(t *testing.T) {
if c, err := ParseConfig(`
[leveldb]
path = "/path/to/db"
`); err != nil {
t.Fatal(err)
} else if c.LevelDB.Path != "/path/to/db" {
t.Fatalf("unexpected path: %s", c.LevelDB.Path)
}
}
// Ensure the "statsd" config can be parsed.
func TestConfig_Parse_Statsd(t *testing.T) {
if c, err := ParseConfig(`
[statsd]
host = "localhost"
`); err != nil {
t.Fatal(err)
} else if c.Statsd.Host != "localhost" {
t.Fatalf("unexpected host: %s", c.Statsd.Host)
} else if c.Addr != ":80" {
t.Fatalf("unexpected addr: %s", c.Addr)
}
}
@ -167,21 +37,6 @@ path = "/path/to/plugins"
}
}
// Ensure the "etcd" config can be parsed.
func TestConfig_Parse_ETCD(t *testing.T) {
if c, err := ParseConfig(`
[etcd]
hosts = ["127.0.0.1"]
fragment-alloc-lock-ttl = "5m"
`); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(c.ETCD.Hosts, []string{"127.0.0.1"}) {
t.Fatalf("unexpected hosts: %+v", c.ETCD.Hosts)
} else if time.Duration(c.ETCD.FragmentAllocLockTTL) != 5*time.Minute {
t.Fatalf("unexpected fragment alloc lock ttl: %v", c.ETCD.FragmentAllocLockTTL)
}
}
// ParseConfig parses s into a config.
func ParseConfig(s string) (main.Config, error) {
var c main.Config

229
cmd/pilosa/main.go
View file

@ -4,37 +4,64 @@ import (
"flag"
"fmt"
"io"
"log"
"math/rand"
"net"
"net/http"
"os"
"runtime/pprof"
"time"
"github.com/BurntSushi/toml"
log "github.com/cihub/seelog"
"github.com/coreos/go-etcd/etcd"
"github.com/kr/s3/s3util"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/dispatch"
"github.com/umbel/pilosa/executor"
"github.com/umbel/pilosa/hold"
"github.com/umbel/pilosa/statsd"
"github.com/umbel/pilosa/transport"
)
// Build holds the build information passed in at compile time.
var Build string
func init() {
if Build == "" {
Build = "v0.0.0"
}
rand.Seed(time.Now().UTC().UnixNano())
}
func main() {
m := NewMain()
if err := m.Run(os.Args[1:]...); err != nil {
fmt.Fprintln(m.Stderr, err.Error())
os.Exit(-1)
fmt.Fprintf(m.Stderr, "Pilosa %s\n", Build)
// Parse command line arguments.
if err := m.ParseFlags(os.Args[1:]); err != nil {
fmt.Fprintln(m.Stderr, err)
os.Exit(2)
}
// Execute the program.
if err := m.Run(); err != nil {
fmt.Fprintln(m.Stderr, err)
os.Exit(1)
}
// Wait indefinitely.
<-(chan struct{})(nil)
}
// Main represents the main program execution.
type Main struct {
ln net.Listener
// Path to the configuration file.
ConfigPath string
// Configuration options.
Config *Config
// Profiling paths
CPUProfile string
// Standard input/output
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
@ -42,6 +69,9 @@ type Main struct {
// NewMain returns a new instance of Main.
func NewMain() *Main {
return &Main{
Config: NewConfig(),
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
@ -49,31 +79,16 @@ func NewMain() *Main {
// Run executes the main program execution.
func (m *Main) Run(args ...string) error {
defer log.Flush()
logger := log.New(m.Stderr, "", log.LstdFlags)
// Parse command line arguments.
opt, err := m.ParseFlags(args)
if err != nil {
return err
}
// Parse configuration.
config := NewConfig()
if opt.ConfigPath != "" {
if _, err := toml.DecodeFile(opt.ConfigPath, &config); err != nil {
return err
}
}
// Generate an ID if one is not specified in the config.
id := config.ID
if id == nil {
*id = pilosa.NewGUID()
// Notify user of config file.
if m.ConfigPath != "" {
fmt.Fprintf(m.Stdout, "Using config: %s\n", m.ConfigPath)
}
// Set up profiling.
if opt.CPUProfile != "" {
f, err := os.Create(opt.CPUProfile)
if m.CPUProfile != "" {
f, err := os.Create(m.CPUProfile)
if err != nil {
return err
}
@ -82,127 +97,61 @@ func (m *Main) Run(args ...string) error {
defer pprof.StopCPUProfile()
}
// Pass configuration to packages.
// NOTE: This is temporary. These config options should be encapsulated in the types.
db.SupportedFrames = config.Storage.SupportedFrames
pilosa.FragmentBase = config.Storage.FragmentBase
pilosa.Backend = config.Storage.Backend
pilosa.LevelDBPath = config.LevelDB.Path
// Build cluster from config file.
cluster := m.Config.PilosaCluster()
// Initialize AWS storage.
s3util.DefaultConfig.AccessKey = config.AWS.AccessKeyID
s3util.DefaultConfig.SecretKey = config.AWS.SecretAccessKey
// Create index to store fragments.
index := pilosa.NewIndex()
// Initialize Statsd.
statsd.Host = config.Statsd.Host
statsd.Setup()
// Create executor for executing queries.
e := pilosa.NewExecutor(index)
e.Host = m.Config.Host
e.Cluster = cluster
// Initialize logging.
logger, _ := log.LoggerFromConfigAsBytes([]byte(SeelogProductionConfig(config.Log.Path, *id, config.Log.Level)))
log.ReplaceLogger(logger)
// Initialize HTTP handler.
h := pilosa.NewHandler()
h.Executor = e
h.LogOutput = m.Stderr
// Initialize etcd client.
etcdClient := etcd.NewClient(config.ETCD.Hosts)
// Open HTTP listener.
ln, err := net.Listen("tcp", m.Config.Addr)
if err != nil {
return err
}
m.ln = ln
// Initialize the cluster.
cluster := db.NewCluster()
// Serve HTTP.
go func() { logger.Print(http.Serve(ln, h)) }()
// Create index.
idx := pilosa.NewFragmentContainer()
// Initialize the holder.
hold := hold.NewHolder()
// Initialize process map.
processMap := core.NewProcessMap()
// Start process mapper.
processMapper := core.NewProcessMapper("/pilosa/0")
processMapper.ID = *id
processMapper.TCPPort = config.TCP.Port
processMapper.HTTPPort = config.HTTP.Port
processMapper.Host = config.Host
processMapper.ProcessMap = processMap
processMapper.EtcdClient = etcdClient
// Start topology mapper.
topologyMapper := core.NewTopologyMapper("/pilosa/0")
topologyMapper.Cluster = cluster
topologyMapper.ProcessMap = processMap
topologyMapper.EtcdClient = etcdClient
topologyMapper.Index = idx
topologyMapper.SupportedFrames = config.Storage.SupportedFrames
topologyMapper.FragmentAllocLockTTL = time.Duration(config.ETCD.FragmentAllocLockTTL)
// Start the transport.
transport := transport.NewTcpTransport(*id)
transport.Port = config.TCP.Port
transport.ProcessMap = processMap
go transport.Run()
// Create the pinger.
pinger := core.NewPinger(*id)
pinger.Hold = hold
pinger.Transport = transport
// Create the batcher.
batcher := core.NewBatcher(*id)
batcher.Cluster = cluster
batcher.Hold = hold
batcher.Transport = transport
// Start the web service.
core.RequestLogPath = config.HTTP.RequestLogPath
ws := core.NewWebService()
ws.ID = *id
ws.Port = config.HTTP.Port
ws.Version = Build
ws.DefaultDB = config.HTTP.DefaultDB
ws.SetBitLogEnabled = config.HTTP.SetBitLogEnabled
ws.Cluster = cluster
ws.TopologyMapper = topologyMapper
ws.Pinger = pinger
ws.Batcher = batcher
// Start the executor.
ex := executor.NewExecutor(*id)
ex.ProcessMap = processMap
ex.PluginsPath = config.Plugins.Path
ex.Hold = hold
ex.Index = idx
// Start the dispatcher.
dispatch := dispatch.NewDispatch()
dispatch.Executor = ex
dispatch.Hold = hold
dispatch.Index = idx
dispatch.Transport = transport
go dispatch.Run()
fmt.Printf("Pilosa %s\n", Build)
log.Warn("STOP")
fmt.Fprintf(m.Stderr, "Listening on http://%s\n", ln.Addr().String())
return nil
}
// Close shuts down the process.
func (m *Main) Close() error {
if m.ln != nil {
m.ln.Close()
}
return nil
}
// ParseFlags parses command line flags from args.
func (m *Main) ParseFlags(args []string) (Options, error) {
var opt Options
func (m *Main) ParseFlags(args []string) error {
fs := flag.NewFlagSet("pilosa", flag.ContinueOnError)
fs.SetOutput(m.Stderr)
fs.StringVar(&opt.ConfigPath, "config", "", "config path")
fs.StringVar(&opt.CPUProfile, "cpuprofile", "", "write cpu profile to file")
fs.StringVar(&m.ConfigPath, "config", "", "config path")
fs.StringVar(&m.CPUProfile, "cpuprofile", "", "write cpu profile to file")
if err := fs.Parse(args); err != nil {
return opt, err
return err
}
return opt, nil
}
// Load config, if specified.
if m.ConfigPath != "" {
if _, err := toml.DecodeFile(m.ConfigPath, &m.Config); err != nil {
return err
}
}
// Options represents the command line options.
type Options struct {
CPUProfile string
ConfigPath string
return nil
}

28
cmd/pilosa/seelog.go
View file

@ -1,28 +0,0 @@
package main
import (
"fmt"
"github.com/umbel/pilosa"
)
func SeelogProductionConfig(path string, id pilosa.GUID, level string) string {
if path == "" {
path = "/tmp"
}
fname := fmt.Sprintf("%s/pilosa.%s", path, id.String())
//<seelog minlevel="debug" maxlevel="error">
s := fmt.Sprintf(`<seelog minlevel="%s">
<outputs>
<rollingfile type="size" filename="%s" maxsize="524288000" maxrolls="4" formatid="format1" />
</outputs> `, level, fname)
s += `<formats>
<format id="format1" format="%Date/%Time [%LEV] %Msg%n"/>
</formats>
</seelog>`
return s
}

65
cmd/pilosactl/main.go Normal file
View file

@ -0,0 +1,65 @@
package main
import (
"errors"
"flag"
"fmt"
"io"
"os"
)
func main() {
m := NewMain()
// Parse command line arguments.
if err := m.ParseFlags(os.Args[1:]); err != nil {
fmt.Fprintln(m.Stderr, err)
os.Exit(2)
}
// Execute the program.
if err := m.Run(); err != nil {
fmt.Fprintln(m.Stderr, err)
os.Exit(1)
}
}
// Main represents the main program execution.
type Main struct {
Command string
// Standard input/output
Stdin io.Reader
Stdout io.Writer
Stderr io.Writer
}
// NewMain returns a new instance of Main.
func NewMain() *Main {
return &Main{
Stdin: os.Stdin,
Stdout: os.Stdout,
Stderr: os.Stderr,
}
}
// Run executes the main program execution.
func (m *Main) Run() error {
panic("FIXME: implement commands")
}
// ParseFlags parses command line flags from args.
func (m *Main) ParseFlags(args []string) error {
fs := flag.NewFlagSet("pilosactl", flag.ContinueOnError)
fs.SetOutput(m.Stderr)
if err := fs.Parse(args); err != nil {
return err
}
// Require command.
if fs.NArg() == 0 {
return errors.New("command required")
}
return nil
}

1
cmd/pilosactl/main_test.go Normal file
View file

@ -0,0 +1 @@
package main_test

70
core/batch.go
View file

@ -1,70 +0,0 @@
package core
import (
"encoding/gob"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/hold"
)
type BatchRequest struct {
Id *pilosa.GUID
Source *pilosa.GUID
Fragment_id pilosa.SUUID
Bitmap_id uint64
Compressed_bitmap string
Filter uint64
}
type BatchResponse struct {
Id *pilosa.GUID
}
func (self BatchResponse) ResultId() *pilosa.GUID {
return self.Id
}
func (self BatchResponse) ResultData() interface{} {
return self.Id
}
func init() {
gob.Register(BatchRequest{})
gob.Register(BatchResponse{})
}
type Batcher struct {
ID pilosa.GUID
Cluster *db.Cluster
Hold *hold.Holder
Transport interface {
Send(message *db.Message, host *pilosa.GUID)
}
}
func NewBatcher(id pilosa.GUID) *Batcher {
return &Batcher{ID: id}
}
func (b *Batcher) Batch(database_name, frame, compressed_bitmap string, bitmap_id uint64, slice int, filter uint64) error {
log.Trace("Batch:", "db:", database_name, " frame:", frame, " slice:", slice, " cb:", compressed_bitmap, " bid:", bitmap_id, "f:", filter)
//determine the fragment_id from database/frame/slice
database := b.Cluster.GetOrCreateDatabase(database_name)
oslice := database.GetOrCreateSlice(slice)
//need to find processid and fragment id for that slice
fragment, err := database.GetFragmentForBitmap(oslice, &db.Bitmap{Id: bitmap_id, FrameType: frame, Filter: filter})
if err == nil {
id := pilosa.NewGUID()
batch := db.Message{Data: BatchRequest{Id: &id, Source: &b.ID, Fragment_id: fragment.GetId(), Bitmap_id: bitmap_id, Compressed_bitmap: compressed_bitmap}}
dest_id := fragment.GetProcess().Id()
b.Transport.Send(&batch, &dest_id)
_, err = b.Hold.Get(&id, 60)
}
return err
}

519
core/etcd.go
View file

@ -1,519 +0,0 @@
package core
import (
"errors"
"fmt"
"os"
"runtime/debug"
"sort"
"strconv"
"strings"
"sync"
"time"
log "github.com/cihub/seelog"
"github.com/coreos/go-etcd/etcd"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
const (
DefaultFragmentAllocLockTTL = 14400 * time.Second
)
type TopologyMapper struct {
namespace string
ID pilosa.GUID
Cluster *db.Cluster
ProcessMap *ProcessMap
SupportedFrames []string
FragmentAllocLockTTL time.Duration
EtcdClient interface {
CreateDir(key string, ttl uint64) (*etcd.Response, error)
Get(key string, sort, recursive bool) (*etcd.Response, error)
RawCreate(key string, value string, ttl uint64) (*etcd.RawResponse, error)
Set(key string, value string, ttl uint64) (*etcd.Response, error)
Watch(prefix string, waitIndex uint64, recursive bool, receiver chan *etcd.Response, stop chan bool) (*etcd.Response, error)
}
Index interface {
AddFragment(db string, frame string, slice int, id pilosa.SUUID)
}
}
func NewTopologyMapper(namespace string) *TopologyMapper {
return &TopologyMapper{
namespace: namespace,
FragmentAllocLockTTL: DefaultFragmentAllocLockTTL,
}
}
func (self *TopologyMapper) Setup() {
log.Warn(self.namespace + "/db")
db_path := self.namespace + "/db"
resp, err := self.EtcdClient.Get(db_path, false, true)
if err != nil {
ee, ok := err.(*etcd.EtcdError)
if ok && ee.ErrorCode == 100 { // node does not exist
resp, err = self.EtcdClient.CreateDir(db_path, 0)
if err != nil {
log.Critical(err)
os.Exit(-1)
}
} else {
log.Critical(err)
os.Exit(-1)
}
}
//need to lock the world
for _, node := range flatten(resp.Node) {
err := self.handlenode(node)
if err != nil {
log.Warn(err)
}
}
}
func (self *TopologyMapper) Run() {
receiver := make(chan *etcd.Response)
go func() {
// TODO: add some terminating measure
// TODO: use modindex to make sure watch catches everything
for {
ns := self.namespace + "/db"
log.Warn(" ETCD watcher:", ns)
stop := make(chan bool)
resp, err := self.EtcdClient.Watch(ns, 0, true, receiver, stop)
log.Warn("TopologyMapper ETCD watcher", resp, err)
}
}()
go func() {
for resp := range receiver {
switch resp.Action {
case "set":
self.handlenode(resp.Node)
case "delete":
self.remove_fragment(resp.Node)
}
// TODO: handle deletes
}
}()
}
type Pair struct {
Key string
Value int
}
// A slice of Pairs that implements sort.Interface to sort by Value.
type PairList []Pair
func (p PairList) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p PairList) Len() int { return len(p) }
func (p PairList) Less(i, j int) bool { return p[i].Value < p[j].Value }
func getLightestProcess(m map[string]int) (Pair, error) {
l := len(m)
if l == 0 {
return Pair{}, errors.New("No Processes")
}
processlist := make(PairList, l)
i := 0
for k, v := range m {
processlist[i] = Pair{k, v}
i++
}
sort.Sort(processlist)
return processlist[0], nil
}
func (self *TopologyMapper) GetProcessFragmentCounts() map[string]int {
m := make(map[string]int)
m[self.ID.String()] = 0 //at least have one process if none created
for k, _ := range self.ProcessMap.nodes {
p := k.String()
m[p] = 0 //at least have one process if none created
}
for _, dbs := range self.Cluster.GetDatabases() {
for _, fsi := range dbs.GetFrameSliceIntersects() {
for _, fragment := range fsi.GetFragments() {
process := fragment.GetProcess().Id().String()
if len(process) > 1 {
i := m[process]
i++
m[process] = i
}
}
}
}
return m
}
func (self *TopologyMapper) MakeFragments(db string, slice_int int) error {
lock_key := fmt.Sprintf("%s/lock/%s-%d", self.namespace, db, slice_int)
response, err := self.EtcdClient.RawCreate(lock_key, "0", uint64(self.FragmentAllocLockTTL.Seconds()))
if err == nil {
if response.StatusCode == 201 { //key created
log.Warn("MakeFragments:", db, slice_int)
m := self.GetProcessFragmentCounts()
p, err := getLightestProcess(m)
if err != nil {
log.Warn("MakeFragments: error finding process", db, slice_int, err)
return err
}
// PUT -d "value=5cb315c3-6e1d-4218-89b7-943d1dba985b" http://etcd0:4001/v2/keys/pilosa/0/db/29/frame/d/slice/5/fragment/a2b632fc4001b817/proces
for _, frame := range self.SupportedFrames {
err := self.AllocateFragment(p.Key, db, frame, slice_int)
if err != nil {
log.Warn(err)
}
}
}
}
return nil
}
func (self *TopologyMapper) AllocateFragment(process_guid, db, frame string, slice_int int) error {
//get Lock to create the fragment
//figure out least loaded process..possibly check max process
//to create the node, just write off the items to etcd and the watch should spawn
//be nice if something would notify perhaps queue
//so i need db, frame, slice , fragment_id
fuid := pilosa.NewSUUID().String()
fragment_key := fmt.Sprintf("%s/db/%s/frame/%s/slice/%d/fragment/%s/process", self.namespace, db, frame, slice_int, fuid)
// need to check value to see how many we have left
log.Warn("ALLOC:", process_guid, len(process_guid))
if len(process_guid) > 1 {
_, err := self.EtcdClient.Set(fragment_key, process_guid, 0)
if err != nil {
return err
}
log.Warn("Fragment sent to etcd:", fragment_key, process_guid)
}
return nil
}
func (self *TopologyMapper) handlenode(node *etcd.Node) error {
key := node.Key[len(self.namespace)+1:]
bits := strings.Split(key, "/")
var database *db.Database
var frame *db.Frame
var fragment *db.Fragment
var fragment_id pilosa.SUUID
var slice *db.Slice
var slice_int int
var process_uuid pilosa.GUID
var process *db.Process
var err error
if len(bits) > 8 {
if bits[8] != "process" {
return errors.New("no process")
}
process_uuid, err = pilosa.ParseGUID(node.Value)
if err != nil {
log.Warn("Bad Process Guid", key)
return errors.New("No Process Id")
}
} else {
return err
}
if len(bits) <= 1 || bits[0] != "db" {
return nil
}
if len(bits) > 1 {
database = self.Cluster.GetOrCreateDatabase(bits[1])
}
if len(bits) > 2 {
if bits[2] != "frame" {
return errors.New("no frame")
}
}
if len(bits) > 3 {
frame = database.GetOrCreateFrame(bits[3])
}
if len(bits) > 4 {
if bits[4] != "slice" {
return errors.New("no slice")
}
}
if len(bits) > 5 {
slice_int, err = strconv.Atoi(bits[5])
if err != nil {
return err
}
slice = database.GetOrCreateSlice(slice_int)
}
if len(bits) > 6 {
if bits[6] != "fragment" {
return errors.New("no fragment")
}
}
if len(bits) > 7 {
fragment_id = pilosa.ParseSUUID(bits[7])
fragment = database.GetOrCreateFragment(frame, slice, fragment_id)
}
if len(bits) > 8 {
if bits[8] != "process" {
return errors.New("no process")
}
if err != nil {
log.Warn("Bad UUID:", process_uuid, key)
return err
}
process = db.NewProcess(&process_uuid)
fragment.SetProcess(process)
if self.ID.Equals(&process_uuid) {
self.Index.AddFragment(bits[1], bits[3], slice_int, fragment_id)
}
}
return err
}
func (self *TopologyMapper) remove_fragment(node *etcd.Node) error {
log.Warn(" hot remove_fragment (Not Supported yet):", node)
return nil
}
func flatten(node *etcd.Node) []*etcd.Node {
nodes := []*etcd.Node{node}
for i := 0; i < len(node.Nodes); i++ {
nodes = append(nodes, flatten(&node.Nodes[i])...)
}
return nodes
}
type Node struct {
id *pilosa.GUID
ip string
port_tcp int
port_http int
}
type ProcessMap struct {
nodes map[pilosa.GUID]*db.Process
mutex sync.Mutex
}
func NewProcessMap() *ProcessMap {
p := ProcessMap{}
p.nodes = make(map[pilosa.GUID]*db.Process)
return &p
}
func (self *ProcessMap) AddProcess(process *db.Process) {
self.mutex.Lock()
defer self.mutex.Unlock()
self.nodes[process.Id()] = process
}
func (self *ProcessMap) GetProcess(id *pilosa.GUID) (*db.Process, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
if id == nil {
debug.PrintStack()
return nil, errors.New("Nil process")
}
process, ok := self.nodes[*id]
if !ok {
return nil, errors.New("No such process")
}
return process, nil
}
func (self *ProcessMap) GetOrAddProcess(id *pilosa.GUID) *db.Process {
process, err := self.GetProcess(id)
if err != nil {
process = db.NewProcess(id)
self.AddProcess(process)
}
return process
}
func (self *ProcessMap) GetHost(id *pilosa.GUID) (string, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
process, ok := self.nodes[*id]
if !ok {
return "", errors.New("Process does not exist")
}
return process.Host(), nil
}
func (self *ProcessMap) GetPortTcp(id *pilosa.GUID) (int, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
process, ok := self.nodes[*id]
if !ok {
return 0, errors.New("Process does not exist")
}
return process.PortTcp(), nil
}
func (self *ProcessMap) GetPortHttp(id *pilosa.GUID) (int, error) {
self.mutex.Lock()
defer self.mutex.Unlock()
process, ok := self.nodes[*id]
if !ok {
return 0, errors.New("Process does not exist")
}
return process.PortHttp(), nil
}
func (self *ProcessMap) GetMetadata() map[string]map[string]interface{} {
self.mutex.Lock()
defer self.mutex.Unlock()
out := make(map[string]map[string]interface{})
for id, process := range self.nodes {
pdata := make(map[string]interface{})
pdata["host"] = process.Host()
pdata["port_tcp"] = process.PortTcp()
pdata["port_http"] = process.PortHttp()
out[id.String()] = pdata
}
return out
}
type ProcessMapper struct {
receiver chan etcd.Response
commands chan ProcessMapperCommand
namespace string
ID pilosa.GUID
ProcessMap *ProcessMap
TCPPort int
HTTPPort int
Host string
EtcdClient interface {
Get(key string, sort, recursive bool) (*etcd.Response, error)
Set(key string, value string, ttl uint64) (*etcd.Response, error)
Watch(prefix string, waitIndex uint64, recursive bool, receiver chan *etcd.Response, stop chan bool) (*etcd.Response, error)
}
}
func NewProcessMapper(namespace string) *ProcessMapper {
return &ProcessMapper{
receiver: make(chan etcd.Response),
commands: make(chan ProcessMapperCommand),
namespace: namespace,
}
}
type ProcessMapperCommand struct {
key string
}
func getKey(input string) string {
bits := strings.Split(input, "/")
return bits[len(bits)-1]
}
func (self *ProcessMapper) getnode(u *pilosa.GUID) *Node {
return new(Node)
}
func crash_on_error(err error) {
if err != nil {
log.Critical(err)
os.Exit(-1)
}
}
func (self *ProcessMapper) handlenode(node *etcd.Node) error {
var err error
var process *db.Process
key := node.Key[len(self.namespace)+1:]
bits := strings.Split(key, "/")
if len(bits) <= 1 || bits[0] != "process" {
return nil
}
if len(bits) >= 2 {
id_string := bits[1]
id, err := pilosa.ParseGUID(id_string)
if err != nil {
return errors.New("Invalid GUID: " + id_string + " (" + key + ")")
}
process = self.ProcessMap.GetOrAddProcess(&id)
}
if len(bits) >= 3 {
switch bits[2] {
case "port_tcp":
port_tcp, _ := strconv.Atoi(node.Value)
process.SetPortTcp(port_tcp)
case "port_http":
port_http, _ := strconv.Atoi(node.Value)
process.SetPortHttp(port_http)
case "host":
host := node.Value
process.SetHost(host)
}
}
return err
}
func (self *ProcessMapper) Run() {
path := self.namespace + "/process"
self_path := path + "/" + self.ID.String()
log.Warn("Writing configuration to etcd...")
log.Warn(self_path)
var err error
_, err = self.EtcdClient.Set(self_path+"/port_tcp", strconv.Itoa(self.TCPPort), 0)
crash_on_error(err)
_, err = self.EtcdClient.Set(self_path+"/port_http", strconv.Itoa(self.HTTPPort), 0)
crash_on_error(err)
_, err = self.EtcdClient.Set(self_path+"/host", self.Host, 0)
crash_on_error(err)
response, err := self.EtcdClient.Get(path, false, true)
for _, node := range flatten(response.Node) {
err := self.handlenode(node)
if err != nil {
out := spew.Sdump(node)
log.Warn(err, out)
}
}
receiver := make(chan *etcd.Response)
stop := make(chan bool)
go func() {
// TODO: error check and restart watcher
// TODO: use modindex to make sure watch catches everything
_, _ = self.EtcdClient.Watch(path, 0, true, receiver, stop)
}()
go func() {
for response = range receiver {
switch response.Action {
case "set":
self.handlenode(response.Node)
}
// TODO: handle deletes
}
}()
}

854
core/http.go
View file

@ -1,854 +0,0 @@
package core
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/gob"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/httputil"
"os"
"reflect"
"runtime"
"strconv"
"strings"
"time"
notify "github.com/bitly/go-notify"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/gorilla/websocket"
"github.com/kr/s3/s3util"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/statsd"
)
const DefaultRequestLogPath = "/tmp/set_bit_log"
var RequestLogPath = DefaultRequestLogPath
type WebService struct {
end chan bool
ID pilosa.GUID
Version string
Port int
DefaultDB string
SetBitLogEnabled bool
Cluster *db.Cluster
ProcessMap *ProcessMap
Batcher *Batcher
Executor interface {
RunPQL(database_name string, pql string) (interface{}, error)
}
Pinger interface {
Ping(process_id *pilosa.GUID) (*time.Duration, error)
}
TopologyMapper interface {
MakeFragments(db string, slice_int int) error
}
Transport interface {
Push(message *db.Message)
}
}
func NewWebService() *WebService {
return &WebService{
end: make(chan bool),
}
}
type Flusher struct {
flusher chan bool
}
func (self *Flusher) Handle(w http.ResponseWriter, r *http.Request) {
self.flusher <- true
}
func NewFlusher(flusher chan bool) http.HandlerFunc {
f := Flusher{flusher}
return f.Handle
}
type RequestLogger struct {
handler http.HandlerFunc
logger chan []byte
}
func NewRequestLogger(handler http.HandlerFunc, logger chan []byte) http.HandlerFunc {
r := RequestLogger{handler, logger}
return r.Handle
}
func (self *RequestLogger) Handle(w http.ResponseWriter, r *http.Request) {
// Grab a dump of the incoming request
dump, err := httputil.DumpRequest(r, true /*dump the body also*/)
if err != nil {
log.Warn("Dump Failure", err)
}
self.handler(w, r)
self.logger <- dump
}
type LogRecord struct {
When time.Time
Data_x64 string
}
func NewLogRecord(t time.Time, data []byte) LogRecord {
var b bytes.Buffer
w := gzip.NewWriter(&b)
w.Write(data)
w.Flush()
w.Close()
e := base64.StdEncoding.EncodeToString(b.Bytes())
x := LogRecord{t, e}
return x
}
func genFileName(id string) string {
//bucket/YYYY/MM/DDHHMMSS.id.log
t := time.Now()
//base := "http://pilosa.umbel.com.s3.amazonaws.com/bit_log"
return fmt.Sprintf("%s%s.%s.log", RequestLogPath, t.Format("/2006/01/02/15/04-05"), id)
}
func flush(requests []LogRecord, id string, records_to_dump int) {
dest := genFileName(id)
w, err := createFile(dest)
if err != nil {
log.Warn("Error opening outfile ", dest)
log.Warn(err)
return
}
defer w.Close()
encoder := json.NewEncoder(w)
for i := 0; i < records_to_dump; i++ {
encoder.Encode(requests[i])
}
}
func Logger(in chan []byte, end chan bool, id string, flusher chan bool) {
var buffer = make([]LogRecord, 2048, 2048)
i := 0
for {
select {
case raw := <-in:
logRecord := NewLogRecord(time.Now(), raw)
buffer[i] = logRecord
i += 1
if i > 2047 {
flush(buffer, id, i)
i = 0
}
case <-flusher:
if i > 0 {
flush(buffer, id, i)
i = 0
}
case <-end:
flush(buffer, id, i)
log.Info("Shutdown Logger")
return
}
}
}
func (self *WebService) Run() {
port_string := strconv.Itoa(self.Port)
log.Info("Serving HTTP on port:", port_string)
logger_chan := make(chan []byte, 1024)
flusher := make(chan bool)
mux := http.NewServeMux()
mux.HandleFunc("/message", self.HandleMessage)
mux.HandleFunc("/query", self.HandleQuery)
mux.HandleFunc("/stats", self.HandleStats)
mux.HandleFunc("/status", self.HandleStatus)
mux.HandleFunc("/info", self.HandleInfo)
mux.HandleFunc("/processes", self.HandleProcesses)
mux.HandleFunc("/listen/ws", self.HandleListenWS)
mux.HandleFunc("/listen/stream", self.HandleListenStream)
mux.HandleFunc("/listen", self.HandleListen)
mux.HandleFunc("/test", self.HandleTest)
mux.HandleFunc("/version", self.HandleVersion)
mux.HandleFunc("/ping", self.HandlePing)
mux.HandleFunc("/batch", self.HandleBatch)
mux.HandleFunc("/load", self.HandleLoad)
if self.SetBitLogEnabled {
mux.HandleFunc("/set_bits", NewRequestLogger(self.HandleSetBit, logger_chan))
} else {
mux.HandleFunc("/set_bits", self.HandleSetBit)
}
mux.HandleFunc("/clear_bits", self.HandleClearBit)
//mux.HandleFunc("/set_bits", self.HandleSetBit)
mux.HandleFunc("/flush", NewFlusher(flusher))
s := &http.Server{
Addr: ":" + port_string,
Handler: mux,
}
go Logger(logger_chan, self.end, self.ID.String(), flusher)
s.ListenAndServe()
}
func (self *WebService) Shutdown() {
self.end <- true
}
func (self *WebService) HandleMessage(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
var message db.Message
decoder := json.NewDecoder(r.Body)
if decoder.Decode(&message) != nil {
http.Error(w, "Invalid JSON", http.StatusBadRequest)
return
}
}
func (self *WebService) HandleLoad(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
decoder := json.NewDecoder(r.Body)
var obj JsonObject
err := decoder.Decode(&obj)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
database_name, ok := obj["db"]
if !ok {
http.Error(w, "Provide a database (db)", http.StatusNotFound)
return
}
db := database_name.(string)
ms_, ok := obj["max_slice"]
if ok {
ms := int(ms_.(float64))
database := self.Cluster.GetOrCreateDatabase(db)
ns, _ := database.NumSlices()
if ns <= ms {
for i := ns; i <= ms; i++ {
log.Info("Load Create Slice ", i)
self.TopologyMapper.MakeFragments(db, i)
}
http.Error(w, "Needed Slices", http.StatusNotFound)
return
}
}
_, ok = obj["id"]
if !ok {
http.Error(w, "Provide a bitmap id (id)", http.StatusNotFound)
return
}
t := float64(obj["id"].(float64))
bitmap_id := uint64(t)
frame, ok := obj["frame"]
if !ok {
http.Error(w, "Provide a frame (frame)", http.StatusNotFound)
return
}
api_string, ok := obj["bitmap"]
if !ok {
http.Error(w, "Provide a compressed base64 bitmap (bitmap)", http.StatusNotFound)
return
}
_, ok = obj["filter"]
if !ok {
http.Error(w, "Provide a filter for categories", http.StatusNotFound)
return
}
t = float64(obj["filter"].(float64))
filter := uint64(t)
results := FromApiString(self.Batcher, database_name.(string), frame.(string), api_string.(string), bitmap_id, filter)
encoder := json.NewEncoder(w)
err = encoder.Encode(results)
if err != nil {
log.Warn("Error Load results")
log.Warn(spew.Sdump(r.Form))
err = encoder.Encode("Bad Batch Request")
}
}
func (self *WebService) HandleBatch(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
err := r.ParseForm()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
database_name := r.Form.Get("db")
if database_name == "" {
http.Error(w, "Provide a database (db)", http.StatusNotFound)
return
}
bitmap_id_string := r.Form.Get("id")
if bitmap_id_string == "" {
http.Error(w, "Provide a bitmap id (id)", http.StatusNotFound)
return
}
bitmap_id, _ := strconv.ParseUint(bitmap_id_string, 10, 64)
slice_string := r.Form.Get("slice")
if bitmap_id_string == "" {
http.Error(w, "Provide a slice (slice)", http.StatusNotFound)
return
}
slice, _ := strconv.ParseInt(slice_string, 10, 32)
frame := r.Form.Get("frame")
if bitmap_id_string == "" {
http.Error(w, "Provide a frame (frame)", http.StatusNotFound)
return
}
compressed_bitmap := r.Form.Get("bitmap")
if bitmap_id_string == "" {
http.Error(w, "Provide a compressed base64 bitmap (bitmap)", http.StatusNotFound)
return
}
filter_string := r.Form.Get("filter")
filter, _ := strconv.ParseInt(filter_string, 10, 32)
if bitmap_id_string == "" {
http.Error(w, "Provide a filter for categories", http.StatusNotFound)
return
}
results := self.Batcher.Batch(database_name, frame, compressed_bitmap, bitmap_id, int(slice), uint64(filter))
encoder := json.NewEncoder(w)
err = encoder.Encode(results)
if err != nil {
log.Warn("Error Batch results")
log.Warn(spew.Sdump(r.Form))
err = encoder.Encode("Bad Batch Request")
}
}
func (self *WebService) HandleQuery(w http.ResponseWriter, r *http.Request) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
statsd.SendInc("webservice_Query")
err := r.ParseForm()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
database_name := r.Form.Get("db")
if database_name == "" {
database_name = self.DefaultDB
}
if database_name == "" {
http.Error(w, "Provide a database (db)", http.StatusNotFound)
return
}
if !self.Cluster.IsValidDatabase(database_name) {
http.Error(w, "Unknown Database:"+database_name, http.StatusNotFound)
return
}
pql := r.Form.Get("pql")
if pql == "" {
http.Error(w, "Provide a valid query string (pql)", http.StatusNotFound)
return
}
_, bits := r.Form["bits"]
log.Debug("PQL:", database_name, pql)
results, err := self.Executor.RunPQL(database_name, pql)
if err != nil {
log.Warn("PQL Exec Error:", err.Error(), database_name, pql)
http.Error(w, "Error encoding: "+err.Error(), http.StatusInternalServerError)
return
}
switch r := results.(type) { //a hack to handle empty sets
case []pilosa.Pair:
if len(r) == 0 {
results = []int{}
}
case []byte: //can i figure out the type of the compressed string here?
if bits {
reader, _ := gzip.NewReader(bytes.NewReader(r))
b, _ := ioutil.ReadAll(reader)
result := pilosa.NewBitmap()
result.FromBytes(b)
results = result.Bits()
}
}
if results == nil {
log.Warn("Empty results:", database_name, pql)
http.Error(w, "Error encoding: "+err.Error(), http.StatusInternalServerError)
return
}
if err != nil {
http.Error(w, "Error running query: "+err.Error(), http.StatusInternalServerError)
return
}
encoder := json.NewEncoder(w)
err = encoder.Encode(results)
if err != nil {
log.Warn("Encode Error :", database_name, pql, err.Error())
return
}
}
type JsonObject map[string]interface{}
// post this json to the body
//the frame type with extension ".t" are handled a bit differnt
// it generats the timestamp based bitmap_ids
//[{ "db": "3", "frame":"b.n","profile_id": 122,"filter":0, "bitmap_id":123},
// { "db": "3", "frame":"b.n","profile_id": 122,"filter":2, "bitmap_id":124},
// { "db": "3", "frame":"t.t","profile_id": 122,"filter":2, "bitmap_id":124, "timestamp":"2014-04-03 13:01:04"}]'
//
func bitmaps(frame string, obj JsonObject) chan uint64 {
c := make(chan uint64)
go func() {
const shortFormT = "2006-01-02T15:04:05"
const shortFormS = "2006-01-02 15:04:05"
t := float64(obj["bitmap_id"].(float64))
base_id := uint64(t)
if strings.HasSuffix(frame, ".t") {
timestamp, present := obj["timestamp"].(string)
if !present || timestamp == "2014-01-01 00:00:00" { //skip the default timestamp
c <- base_id
} else {
quantum := pilosa.YMDH
if val, ok := obj["time_granularity"]; ok {
switch val {
case "Y":
quantum = pilosa.Y
case "M":
quantum = pilosa.YM
case "D":
quantum = pilosa.YMD
}
}
shortForm := shortFormS
if strings.Contains(timestamp, "T") {
shortForm = shortFormT
}
atime, _ := time.Parse(shortForm, timestamp)
for i, id := range pilosa.GetTimeIds(base_id, atime, quantum) {
c <- id
if i > 10 {
log.Warn("TO MANY TIMEIDS", base_id, atime, quantum)
break
}
}
}
} else {
c <- base_id
}
close(c)
}()
return c
}
type SBResult struct {
Bitmap_id uint64
Frame string
Filter uint64
Profile_id uint64
Result interface{}
}
func init() {
gob.Register(SBResult{})
}
func (self *WebService) HandleClearBit(w http.ResponseWriter, r *http.Request) {
self.HandleBit(w, r, false)
}
func (self *WebService) HandleSetBit(w http.ResponseWriter, r *http.Request) {
self.HandleBit(w, r, true)
}
func (self *WebService) HandleBit(w http.ResponseWriter, r *http.Request, ToSet bool) {
if r.Method != "POST" {
http.Error(w, "Only POST allowed", http.StatusMethodNotAllowed)
return
}
decoder := json.NewDecoder(r.Body)
var args []JsonObject
err := decoder.Decode(&args)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
//[{ "db": "3", "frame":"brand.","profile_id": 122,"filter":0, "bitmap_id":123}]
var results []SBResult
if len(args) > 4096 {
log.Warn("Request too large:", len(args))
http.Error(w, "Request To large", http.StatusBadRequest)
return
}
//remoteSetBit := NewRemoteSetBit()
//remoteSetBit.ID = self.ID
//remoteSetBit.ProcessMap = self.ProcessMap
//remoteSetBit.Hold = self.Hold
//remoteSetBit.Transport = self.Transport
for _, obj := range args {
if obj["profile_id"] == nil {
http.Error(w, "Missing Profile", http.StatusBadRequest)
return
}
t := float64(obj["profile_id"].(float64))
profile_id := uint64(t)
if obj["db"] == nil {
http.Error(w, "Missing db", http.StatusBadRequest)
return
}
dbs := obj["db"].(string)
if obj["frame"] == nil {
http.Error(w, "Missing Frame", http.StatusBadRequest)
return
}
frame := obj["frame"].(string)
if obj["filter"] == nil {
http.Error(w, "Missing Filter", http.StatusBadRequest)
return
}
t = float64(obj["filter"].(float64))
filter := uint64(t)
for bitmap_id := range bitmaps(frame, obj) {
var pql string
if ToSet {
pql = fmt.Sprintf("set(%d, %s, %d, %d)", bitmap_id, frame, filter, profile_id)
} else {
pql = fmt.Sprintf("clear(%d, %s, %d, %d)", bitmap_id, frame, filter, profile_id)
}
result, err := self.Executor.RunPQL(dbs, pql)
bundle := SBResult{bitmap_id, frame, filter, profile_id, result}
results = append(results, bundle)
if err != nil {
log.Warn("Error running set_bit", dbs, frame, profile_id, ToSet)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
}
}
encoder := json.NewEncoder(w)
err = encoder.Encode(results)
if err != nil {
log.Warn("JSON SetBit ERROR:", err, ToSet)
return
}
}
func (self *WebService) HandleStats(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
encoder := json.NewEncoder(w)
m := &runtime.MemStats{}
runtime.ReadMemStats(m)
err := encoder.Encode(m)
if err != nil {
log.Warn("Error encoding stats")
http.Error(w, "Error econding stats", http.StatusMethodNotAllowed)
}
}
func (self *WebService) HandleInfo(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
spew.Fdump(w, self.Cluster)
}
func (self *WebService) HandleVersion(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
fmt.Fprintf(w, "Pilosa v.("+self.Version+")\n")
}
func (self *WebService) HandleTest(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
spew.Dump("TEST!")
msg := new(db.Message)
msg.Data = "mystring"
self.Transport.Push(msg)
msg2 := new(db.Message)
msg2.Data = 789
self.Transport.Push(msg2)
}
func (self *WebService) HandleProcesses(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
encoder := json.NewEncoder(w)
processes := self.ProcessMap.GetMetadata()
err := encoder.Encode(processes)
if err != nil {
http.Error(w, "Error Encoding", http.StatusBadRequest)
}
}
func (self *WebService) HandlePing(w http.ResponseWriter, r *http.Request) {
if r.Method != "GET" {
http.Error(w, "Only GET allowed", http.StatusMethodNotAllowed)
return
}
err := r.ParseForm()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
process_string := r.Form.Get("process")
process_id, err := pilosa.ParseGUID(process_string)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
_, err = self.ProcessMap.GetProcess(&process_id)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
duration, err := self.Pinger.Ping(&process_id)
if err != nil {
spew.Fdump(w, err)
return
}
encoder := json.NewEncoder(w)
encoder.Encode(map[string]float64{"duration": duration.Seconds()})
}
func (self *WebService) HandleListen(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`
<html><head><title>Pilosa - streaming client</title></head><body>
<style type="text/css">
dt.inbox { background-color: #efe; }
dt.outbox { background-color: #eef; }
</style>
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script type="text/javascript">
$dl = $("<dl/>")
$dl.on('click', 'dt', function(e) {
$(this).next().toggle()
})
$("body").append($dl)
ws = new WebSocket("ws://" + window.location.host + "/listen/ws")
ws.onmessage = function(mes) {
obj = JSON.parse(mes.data)
if (obj.host) {
$dl.append('<dt class="outbox">&rarr;' + obj.type + ' (to: ' + obj.host + ')</dt>')
} else {
$dl.append('<dt class="inbox">&larr;' + obj.type + '</dt>')
}
$dl.append('<dd style="display:none;"><pre>' + obj.dump + obj.host + '</pre></dd>')
}
</script>
</body></html>
`))
}
func (self *WebService) streamer(writer func(map[string]interface{}) error) {
inbox := make(chan interface{}, 10)
outbox := make(chan interface{}, 10)
notify.Start("inbox", inbox)
notify.Start("outbox", outbox)
var obj interface{}
var data interface{}
var inmessage *db.Message
var outmessage *db.Envelope
var host string
for {
select {
case obj = <-outbox:
outmessage = obj.(*db.Envelope)
data = outmessage.Message.Data
host = outmessage.Host.String()
case obj = <-inbox:
inmessage = obj.(*db.Message)
data = inmessage.Data
host = ""
}
typ := reflect.TypeOf(data)
err := writer(map[string]interface{}{
"type": typ.String(),
"dump": spew.Sdump(data),
"host": host,
})
if err != nil {
log.Info("stopping")
notify.Stop("inbox", inbox)
notify.Stop("outbox", outbox)
drain(inbox)
drain(outbox)
return
}
}
}
func (self *WebService) HandleListenWS(w http.ResponseWriter, r *http.Request) {
defer func() {
err := recover()
out := spew.Sdump(err)
log.Info(out)
}()
ws, err := websocket.Upgrade(w, r, nil, 1024, 1024)
if _, ok := err.(websocket.HandshakeError); ok {
http.Error(w, "Not a websocket handshake", 400)
return
} else if err != nil {
log.Warn(err)
return
}
self.streamer(func(data map[string]interface{}) error {
return ws.WriteJSON(data)
})
}
func drain(ch chan interface{}) {
for {
switch {
case <-ch:
default:
return
}
}
}
func (self *WebService) HandleListenStream(w http.ResponseWriter, r *http.Request) {
defer func() {
err := recover()
spew.Dump(err)
}()
writer := json.NewEncoder(w)
flusher := w.(http.Flusher)
self.streamer(func(data map[string]interface{}) error {
err := writer.Encode(data)
flusher.Flush()
return err
})
}
func (self *WebService) HandleStatus(w http.ResponseWriter, r *http.Request) {
w.Write([]byte(`
<html><head><title>Pilosa - status</title></head><body>
<style type="text/css">
td {
background: #eee;
}
</style>
<script src="//ajax.googleapis.com/ajax/libs/jquery/1.10.2/jquery.min.js"></script>
<script type="text/javascript">
$table = $("<table><tr><th>process id</th><th>host</th><th>tcp port</th><th>http port</th><th>latency</th></tr></table>")
$("body").append($table)
$.ajax('/processes', {
dataType: "json",
success: function(resp) {
$.each(resp, function(index, value) {
$table.append('<tr><td>' + index + '</td><td>' + value.host + '</td><td>' + value.port_tcp + '</td><td>' + value.port_http + '</td><td><button class="pinger"/></td></tr>')
})
}
})
$table.on('click', 'button.pinger', function(e) {
var $td = $(this).parent()
var $tr = $td.parent()
var id = $tr.children('td').eq(0).text()
$.ajax('/ping?process=' + id, {
dataType: 'json',
success: function(resp) {
$td.html(resp.duration)
}
})
})
</script>
</body></html>
`))
}
func createFile(s string) (io.WriteCloser, error) {
if isURL(s) {
return s3util.Create(s, nil, nil)
}
return os.Create(s)
}
func isURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
}

95
core/load.go
View file

@ -1,95 +0,0 @@
package core
// #cgo CFLAGS:-mpopcnt
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/binary"
"errors"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
)
func copy_raw(src [32]uint64) pilosa.Blocks {
o := make(pilosa.Blocks, 32)
for k, v := range src {
o[k] = v
}
return o
}
func sendBitmap(batcher *Batcher, bitmap *pilosa.Bitmap, db string, frame string, bitmap_id, filter uint64, slice int, finish chan error) {
if slice < 0 {
log.Warn("Bad split", db, frame, slice, bitmap_id)
finish <- errors.New("BadSplit")
return
}
// Compress bitmap.
var b bytes.Buffer
w := gzip.NewWriter(&b)
w.Write(bitmap.ToBytes())
w.Flush()
w.Close()
buf := base64.StdEncoding.EncodeToString(b.Bytes())
results := batcher.Batch(db, frame, buf, bitmap_id, slice, filter)
finish <- results
}
func FromApiString(batcher *Batcher, db string, frame string, api_string string, bitmap_id, filter uint64) string {
compressed_data, err := base64.StdEncoding.DecodeString(api_string)
if err != nil {
log.Warn(err)
return "Bad"
}
reader, err := gzip.NewReader(bytes.NewReader(compressed_data))
if err != nil {
log.Warn(err)
return "Bad"
}
var numChunks uint64
err = binary.Read(reader, binary.LittleEndian, &numChunks)
if err != nil {
log.Warn(err)
return "Bad"
}
first := true
bitmap := pilosa.NewBitmap()
last_slice := pilosa.CounterMask
sent_count := 0
finish := make(chan error)
for i := uint64(0); i < numChunks; i++ {
var raw struct {
Key uint64
Block [32]uint64
}
binary.Read(reader, binary.LittleEndian, &raw)
slice := raw.Key >> 5
if slice != last_slice {
if first {
first = false
} else {
//make async later
sent_count += 1
go sendBitmap(batcher, bitmap, db, frame, bitmap_id, filter, int(last_slice), finish)
bitmap = pilosa.NewBitmap()
}
last_slice = slice
}
o := copy_raw(raw.Block)
chunk := &pilosa.Chunk{Key: raw.Key, Value: o}
bitmap.AddChunk(chunk)
}
sent_count += 1
go sendBitmap(batcher, bitmap, db, frame, bitmap_id, filter, int(last_slice), finish)
for i := 0; i < sent_count; i++ {
<-finish
}
return "OK"
}

62
core/ping.go
View file

@ -1,62 +0,0 @@
package core
import (
"encoding/gob"
"time"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
type PingRequest struct {
Id *pilosa.GUID
Source *pilosa.GUID
}
type PongRequest struct {
Id *pilosa.GUID
}
func (self PongRequest) ResultId() *pilosa.GUID {
return self.Id
}
func (self PongRequest) ResultData() interface{} {
return self.Id
}
func init() {
gob.Register(PingRequest{})
gob.Register(PongRequest{})
}
type Pinger struct {
ID pilosa.GUID
Hold interface {
Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error)
}
Transport interface {
Send(message *db.Message, host *pilosa.GUID)
}
}
func NewPinger(id pilosa.GUID) *Pinger {
return &Pinger{
ID: id,
}
}
func (self *Pinger) Ping(process_id *pilosa.GUID) (*time.Duration, error) {
id := pilosa.NewGUID()
ping := db.Message{Data: PingRequest{Id: &id, Source: &self.ID}}
start := time.Now()
self.Transport.Send(&ping, process_id)
_, err := self.Hold.Get(&id, 60*time.Second)
if err != nil {
return nil, err
}
end := time.Now()
dur := end.Sub(start)
return &dur, nil
}

127
core/remotebits.go
View file

@ -1,127 +0,0 @@
package core
import (
"encoding/gob"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
type RemoteSetBit struct {
requests []remote_task
cluster map[*pilosa.GUID][]BitmapRequestItem
ID pilosa.GUID
ProcessMap *ProcessMap
Hold interface {
Get(id *pilosa.GUID, timeout int) (interface{}, error)
}
Transport interface {
Send(message *db.Message, host *pilosa.GUID)
}
}
func init() {
gob.Register(BitmapRequestItem{})
gob.Register(BitsRequest{})
gob.Register(BitsResponse{})
}
type BitsRequest struct {
Bits []BitmapRequestItem
ReturnProcessId pilosa.GUID
QueryId pilosa.GUID
DestProcessId pilosa.GUID
}
type BitmapRequestItem struct {
Fragment_id pilosa.SUUID
Bitmap_id uint64
Profile_id uint64
Filter uint64
Frame string
SetUnset bool
}
func NewRemoteSetBit() *RemoteSetBit {
obj := new(RemoteSetBit)
obj.cluster = make(map[*pilosa.GUID][]BitmapRequestItem)
return obj
}
func (self *RemoteSetBit) Request() {
self.requests = make([]remote_task, 0)
source_process, _ := self.ProcessMap.GetProcess(&self.ID)
for process, request := range self.cluster {
random_id := pilosa.NewGUID()
msg := new(db.Message)
msg.Data = BitsRequest{
Bits: request,
ReturnProcessId: source_process.Id(),
QueryId: random_id,
DestProcessId: *process,
}
wait := len(request) * 10
if wait < 10 {
wait = 10
}
self.requests = append(self.requests, remote_task{random_id, wait})
self.Transport.Send(msg, process)
}
}
type remote_task struct {
id pilosa.GUID
wait_time int
}
func (self *RemoteSetBit) MergeResults(local_results []SBResult) []SBResult {
answers := make(chan []SBResult)
for _, task := range self.requests {
go func(task remote_task) {
value, err := self.Hold.Get(&task.id, task.wait_time) //eiher need to be the frame process or the handler process?
if value == nil {
log.Warn("Bad RemoteSetBit Result:", err)
empty := make([]SBResult, 0, 0)
answers <- empty
} else {
answers <- value.([]SBResult)
}
}(task)
}
for i := 0; i < len(self.requests); i++ {
batch := <-answers
for _, item := range batch {
local_results = append(local_results, item)
}
}
close(answers)
return local_results
}
func (self *RemoteSetBit) Add(frag *db.Fragment, bitmap_id, profile_id, filter uint64, frame string, SetUnset bool) {
x, found := self.cluster[frag.GetProcessId()]
if !found {
x = make([]BitmapRequestItem, 0)
}
x = append(x, BitmapRequestItem{frag.GetId(), bitmap_id, profile_id, filter, frame, SetUnset})
self.cluster[frag.GetProcessId()] = x
}
type BitsResponse struct {
Id *pilosa.GUID
Items []SBResult
}
func (self *BitsResponse) ResultId() *pilosa.GUID {
return self.Id
}
func (self *BitsResponse) ResultData() interface{} {
return self.Items
}

37
core/stopper.go
View file

@ -1,37 +0,0 @@
package core
import (
"sync"
)
type Stopper struct {
TermChans []chan int
DoneChans []chan int
Mutex sync.RWMutex
}
func (stopper *Stopper) Stop() {
var i chan int
var o chan int
stopper.Mutex.RLock()
for _, i = range stopper.TermChans {
go func(i chan int) {
i <- 1
}(i)
}
for _, o = range stopper.DoneChans {
<-o
}
stopper.Mutex.RUnlock()
return
}
func (stopper *Stopper) GetExitChannels() (chan int, chan int) {
termchan := make(chan int, 1)
donechan := make(chan int, 1)
stopper.Mutex.Lock()
stopper.TermChans = append(stopper.TermChans, termchan)
stopper.DoneChans = append(stopper.DoneChans, donechan)
stopper.Mutex.Unlock()
return termchan, donechan
}

3
db/constants.go
View file

@ -1,3 +0,0 @@
package db
const SLICE_WIDTH = 65536

25
db/db.go
View file

@ -1,25 +0,0 @@
package db
import (
"encoding/gob"
"github.com/umbel/pilosa"
)
type Message struct {
Data interface{} `json:"data"`
}
type Envelope struct {
Message *Message
Host *pilosa.GUID
}
type HoldResult interface {
ResultId() *pilosa.GUID
ResultData() interface{}
}
func init() {
gob.Register(Message{})
}

446
db/topology.go
View file

@ -1,446 +0,0 @@
package db
import (
"errors"
"fmt"
"sync"
log "github.com/cihub/seelog"
"github.com/stathat/consistent"
"github.com/umbel/pilosa"
)
// SupportedFrames is a list of frame types that are supported.
var SupportedFrames = []string{"default"}
var FrameDoesNotExistError = errors.New("Frame does not exist.")
var InvalidFrameError = errors.New("Invalid frame.")
var SliceDoesNotExistError = errors.New("Slice does not exist.")
var FragmentDoesNotExistError = errors.New("Fragment does not exist.")
var FrameSliceIntersectDoesNotExistError = errors.New("FrameSliceIntersect does not exist.")
type Location struct {
ProcessId *pilosa.GUID
FragmentId pilosa.SUUID
}
type Process struct {
id *pilosa.GUID
host string
port_tcp int
port_http int
mutex sync.Mutex
}
func NewProcess(id *pilosa.GUID) *Process {
return &Process{id: id}
}
func (self *Process) Id() pilosa.GUID {
self.mutex.Lock()
defer self.mutex.Unlock()
return *self.id
}
func (self *Process) Host() string {
self.mutex.Lock()
defer self.mutex.Unlock()
return self.host
}
func (self *Process) SetHost(host string) {
self.mutex.Lock()
defer self.mutex.Unlock()
self.host = host
}
func (self *Process) PortTcp() int {
self.mutex.Lock()
defer self.mutex.Unlock()
return self.port_tcp
}
func (self *Process) SetPortTcp(port int) {
self.mutex.Lock()
defer self.mutex.Unlock()
self.port_tcp = port
}
func (self *Process) PortHttp() int {
self.mutex.Lock()
defer self.mutex.Unlock()
return self.port_http
}
func (self *Process) SetPortHttp(port int) {
self.mutex.Lock()
defer self.mutex.Unlock()
self.port_http = port
}
/////////// CLUSTERS
//////////////////////////////////////////////////////////////////////
// Represents the entire cluster, and a reference to the Node this instance is
// running on
type Cluster struct {
databases map[string]*Database
mutex sync.Mutex
}
func NewCluster() *Cluster {
cluster := Cluster{}
cluster.databases = make(map[string]*Database)
return &cluster
}
func (self *Cluster) GetDatabases() map[string]*Database {
return self.databases
}
func (self *Cluster) IsValidDatabase(dbname string) bool {
for name, _ := range self.GetDatabases() {
if name == dbname {
return true
}
}
return false
}
/////////// DATABASES
//////////////////////////////////////////////////////////////////////
// A database is a collection of all the frames within a given profile space
type Database struct {
Name string
frames []*Frame
slices []*Slice
frame_slice_intersects []*FrameSliceIntersect
mutex sync.Mutex
}
func (self *Database) GetFrameSliceIntersects() []*FrameSliceIntersect {
return self.frame_slice_intersects
}
// Add a database to a cluster
func (c *Cluster) addDatabase(name string) *Database {
database := Database{Name: name}
if c.databases == nil {
c.databases = make(map[string]*Database)
}
c.databases[name] = &database
return &database
}
func (c *Cluster) getDatabase(name string) (*Database, error) {
value, ok := c.databases[name]
if !ok {
return nil, errors.New("The database does not exist!")
} else {
return value, nil
}
}
func (c *Cluster) GetOrCreateDatabase(name string) *Database {
c.mutex.Lock()
defer c.mutex.Unlock()
database, err := c.getDatabase(name)
if err == nil {
return database
}
return c.addDatabase(name)
}
func stringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
func (d *Database) IsValidFrame(name string) bool {
return stringInSlice(name, SupportedFrames)
}
// Count the number of slices in a database
func (d *Database) NumSlices() (int, error) {
if len(d.slices) < 1 {
return 0, errors.New("Database is empty")
}
return len(d.slices), nil
}
// return the slice_ids that are in a database
func (d *Database) SliceIds() ([]int, error) {
var rtn []int
for _, slice := range d.slices {
rtn = append(rtn, slice.id)
}
return rtn, nil
}
///////// FRAMES
//////////////////////////////////////////////////////////////////////
// A frame is a collection of slices in a given category
// (brands, demographics, etc), specific to a database
type Frame struct {
name string
}
// Get a frame from a database
func (d *Database) getFrame(name string) (*Frame, error) {
// should we check here for supported frames?
if !d.IsValidFrame(name) {
return nil, InvalidFrameError
}
for _, frame := range d.frames {
if frame.name == name {
return frame, nil
}
}
return nil, FrameDoesNotExistError
}
// Add a frame to a database
func (d *Database) addFrame(name string) *Frame {
frame := Frame{name: name}
d.frames = append(d.frames, &frame)
// add intersections
for _, slice := range d.slices {
d.AddFrameSliceIntersect(&frame, slice)
}
return &frame
}
func (d *Database) GetOrCreateFrame(name string) *Frame {
d.mutex.Lock()
defer d.mutex.Unlock()
frame, err := d.getFrame(name)
if err == nil {
return frame
}
return d.addFrame(name)
}
///////// SLICES
///////////////////////////////////////////////////////////////////////////
// A slice is the vertical combination of every fragment.
type Slice struct {
id int
}
func (self *Slice) Id() int {
return self.id
}
// Get a slice from a database
func (d *Database) getSlice(slice_id int) (*Slice, error) {
for _, slice := range d.slices {
if slice.id == slice_id {
return slice, nil
}
}
return nil, SliceDoesNotExistError
}
// Add a slice to a database
func (d *Database) addSlice(slice_id int) *Slice {
slice := Slice{id: slice_id}
d.slices = append(d.slices, &slice)
// add intersections
for _, frame := range d.frames {
d.AddFrameSliceIntersect(frame, &slice)
}
return &slice
}
func (d *Database) GetOrCreateSlice(slice_id int) *Slice {
d.mutex.Lock()
defer d.mutex.Unlock()
slice, err := d.getSlice(slice_id)
if err == nil {
return slice
}
return d.addSlice(slice_id)
}
///////// FRAME-SLICE INTERSECT
////////////////////////////////////////////////////////////////
type FrameSliceIntersect struct {
frame *Frame
slice *Slice
fragments []*Fragment
hashring *consistent.Consistent
}
func (d *Database) AddFrameSliceIntersect(frame *Frame, slice *Slice) *FrameSliceIntersect {
frameslice := FrameSliceIntersect{frame: frame, slice: slice}
d.frame_slice_intersects = append(d.frame_slice_intersects, &frameslice)
frameslice.hashring = consistent.New()
frameslice.hashring.NumberOfReplicas = 16
return &frameslice
}
func (d *Database) GetFrameSliceIntersect(frame *Frame, slice *Slice) (*FrameSliceIntersect, error) {
for _, frameslice := range d.frame_slice_intersects {
if frameslice.frame == frame && frameslice.slice == slice {
return frameslice, nil
}
}
log.Warn("Missing FrameSliceIntersect:", d.Name, frame, slice)
return nil, FrameSliceIntersectDoesNotExistError
}
func (self *FrameSliceIntersect) GetFragments() []*Fragment {
return self.fragments
}
func (d *Database) GetFragment(fragment_id pilosa.SUUID) (*Fragment, error) {
for _, fsi := range d.frame_slice_intersects {
f, err := fsi.GetFragment(fragment_id)
if err == nil {
return f, nil
}
}
return nil, FragmentDoesNotExistError
}
func (self *FrameSliceIntersect) GetFragment(fragment_id pilosa.SUUID) (*Fragment,
error) {
for _, fragment := range self.fragments {
if fragment.id == fragment_id {
return fragment, nil
}
}
return nil, FragmentDoesNotExistError
}
func (self *FrameSliceIntersect) AddFragment(fragment *Fragment) {
self.fragments = append(self.fragments, fragment)
self.hashring.Add(fragment.id.String())
}
///////// FRAGMENTS
//////////////////////////////////////////////////////////////////////////
// A fragment is a collection of bitmaps within a slice. The fragment contains a
// reference to the responsible node for that fragment. The node is in the form
// ip:port
type Fragment struct {
id pilosa.SUUID
process *Process
}
func (self *Fragment) GetId() pilosa.SUUID {
return self.id
}
func (self *Fragment) GetProcess() *Process {
return self.process
}
func (self *Fragment) GetProcessId() *pilosa.GUID {
return self.process.id
}
func (self *Fragment) GetLocation() *Location {
return &Location{self.process.id, self.id}
}
// rename this one
func (d *Database) GetFragmentForBitmap(slice *Slice, bitmap *Bitmap) (*Fragment, error) {
frame, err := d.getFrame(bitmap.FrameType)
if err != nil {
log.Warn("Missing FrameType", bitmap.FrameType, d.Name, slice)
log.Warn(err)
return nil, err
}
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn("Missing frame,slice", frame, slice)
log.Warn(err)
return nil, err
}
frag_id_s, err := fsi.hashring.Get(fmt.Sprintf("%d", bitmap.Id))
if err != nil {
log.Warn("ERROR FSI.GET:", bitmap.Id, bitmap.FrameType, d.Name, frame, slice)
log.Warn(err)
return nil, err
}
frag_id := pilosa.ParseSUUID(frag_id_s)
return fsi.GetFragment(frag_id)
}
func (d *Database) GetFragmentForFrameSlice(frame *Frame, slice *Slice) (*Fragment, error) {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn("Missing frame,slice", frame, slice)
log.Warn(err)
return nil, err
}
frag_id_s, err := fsi.hashring.Get("0")
// we don't need a specific bitmap in here because we're assuming the hashring only has a single element
if err != nil {
log.Warn("ERROR FSI.GET:", d.Name, frame, slice)
log.Warn(err)
return nil, err
}
frag_id := pilosa.ParseSUUID(frag_id_s)
return fsi.GetFragment(frag_id)
}
func (d *Database) getFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) (*Fragment, error) {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn(err)
return nil, err
}
return fsi.GetFragment(fragment_id)
}
func (d *Database) addFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) *Fragment {
fsi, err := d.GetFrameSliceIntersect(frame, slice)
if err != nil {
log.Warn("database.addFragment", err)
return nil
}
fragment := Fragment{id: fragment_id}
fsi.AddFragment(&fragment)
return &fragment
}
func (d *Database) GetOrCreateFragment(frame *Frame, slice *Slice, fragment_id pilosa.SUUID) *Fragment {
d.mutex.Lock()
defer d.mutex.Unlock()
fragment, err := d.getFragment(frame, slice, fragment_id)
if err == nil {
return fragment
}
return d.addFragment(frame, slice, fragment_id)
}
func (f *Fragment) SetProcess(process *Process) {
f.process = process
}
func GetSlice(profile_id uint64) int {
return int(profile_id / SLICE_WIDTH)
}
///////////////////////////////////////////////////////////////////////////////////////////////
// Get a slice from a database
func (d *Database) GetSliceForProfile(profile_id uint64) (*Slice, error) {
return d.getSlice(GetSlice(profile_id))
}
type Bitmap struct {
Id uint64
FrameType string
Filter uint64
}

17
db/topology_test.go
View file

@ -1,17 +0,0 @@
package db_test
import (
"testing"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/util"
)
func TestCluster(t *testing.T) {
c := db.NewCluster()
d := c.GetOrCreateDatabase("main")
f := d.GetOrCreateFrame("general")
sl := d.GetOrCreateSlice(0)
d.GetOrCreateFragment(f, sl, util.Id())
}

123
dispatch/dispatch.go
View file

@ -1,123 +0,0 @@
package dispatch
import (
"time"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/executor"
"github.com/umbel/pilosa/query"
)
type Dispatch struct {
Executor interface {
NewJob(job *db.Message)
}
Hold interface {
Set(id *pilosa.GUID, value interface{}, timeout time.Duration)
}
Index interface {
ClearBit(fragID pilosa.SUUID, bitmapID uint64, pos uint64) (bool, error)
LoadBitmap(fragID pilosa.SUUID, bitmapID uint64, compressedBitmap string, filter uint64)
SetBit(fragID pilosa.SUUID, bitmapID uint64, pos uint64, category uint64) (bool, error)
TopFillBatch(args []pilosa.FillArgs) ([]pilosa.Pair, error)
}
Transport interface {
Receive() *db.Message
Send(message *db.Message, host *pilosa.GUID)
}
}
func NewDispatch() *Dispatch {
return &Dispatch{}
}
func (self *Dispatch) Init() error {
log.Warn("Starting Dispatcher")
return nil
}
func (self *Dispatch) Close() {
log.Warn("Shutting down Dispatcher")
}
// The Local Route
func (d *Dispatch) Run() {
log.Warn("Dispatch Run...")
for {
message := d.Transport.Receive()
switch data := message.Data.(type) {
case core.BatchRequest:
log.Trace("Dispatch.Run BatchRequest")
response := db.Message{Data: core.BatchResponse{Id: data.Id}}
d.Index.LoadBitmap(data.Fragment_id, data.Bitmap_id, data.Compressed_bitmap, data.Filter)
d.Transport.Send(&response, data.Source)
case core.BitsRequest:
log.Trace("Dispatch.Run BitsRequest")
var results []core.SBResult
result := false
for _, v := range data.Bits {
if v.SetUnset {
result, _ = d.Index.SetBit(v.Fragment_id, v.Bitmap_id, v.Profile_id, uint64(v.Filter))
} else {
result, _ = d.Index.ClearBit(v.Fragment_id, v.Bitmap_id, v.Profile_id)
}
bundle := core.SBResult{Bitmap_id: v.Bitmap_id, Frame: v.Frame, Filter: v.Filter, Profile_id: v.Profile_id, Result: result}
results = append(results, bundle)
}
response := db.Message{Data: core.BitsResponse{Id: &data.QueryId, Items: results}}
d.Transport.Send(&response, &data.ReturnProcessId)
case core.PingRequest:
log.Trace("Dispatch.Run Ping")
pong := db.Message{Data: core.PongRequest{Id: data.Id}}
d.Transport.Send(&pong, data.Source)
case db.HoldResult:
log.Trace("Dispatch.Run HoldResult")
d.Hold.Set(data.ResultId(), data.ResultData(), 30*time.Second)
case query.PortableQueryStep:
log.Trace("Dispatch.Run PortableQueryStep")
go d.Executor.NewJob(message)
case executor.TopFill:
log.Trace("Dispatch.Run TopFill")
go d.topFillHandler(message)
case core.BitsResponse:
d.Hold.Set(data.ResultId(), data.ResultData(), 30*time.Second)
default:
spew.Dump(data)
log.Warn("Unprocessed message", data)
}
}
}
func (d *Dispatch) topFillHandler(m *db.Message) {
topfill := m.Data.(executor.TopFill)
topn, err := d.Index.TopFillBatch(topfill.Args)
if err != nil {
log.Warn("TopFillHandler:", err)
}
d.Transport.Send(&db.Message{
Data: query.FillResult{
BaseQueryResult: &query.BaseQueryResult{
Id: &topfill.QueryId,
Data: topn,
},
},
}, &topfill.ReturnProcessId)
}

17
etc/default.yaml
View file

@ -1,17 +0,0 @@
id: 0d92507d-1646-44f8-be81-79d79c1df623
host: localhost
port_tcp: 12001
port_http: 15001
log_path: /tmp
fragment_base: /tmp/single
storage_backend: cassandra
cassandra_host: localhost
cassandra_keyspace: pilosa
supported_frames:
- b.n
- t.t
- l.n
- d
- p.n
etcd_servers:
- http://127.0.0.1:4001

316
executor.go Normal file
View file

@ -0,0 +1,316 @@
package pilosa
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa/internal"
"github.com/umbel/pilosa/pql"
)
// DefaultFrame is the frame used if one is not specified.
const DefaultFrame = "general"
// ErrDatabaseRequired is returned when no database is specified.
var ErrDatabaseRequired = errors.New("database required")
// Executor recursively executes calls in a PQL query across all slices.
type Executor struct {
index *Index
// Local hostname.
Host string
// Cluster configuration
Cluster *Cluster
// Client used for remote HTTP requests.
HTTPClient *http.Client
}
// NewExecutor returns a new instance of Executor.
func NewExecutor(index *Index) *Executor {
return &Executor{
index: index,
HTTPClient: http.DefaultClient,
}
}
// Index returns the index that the executor runs against.
func (e *Executor) Index() *Index { return e.index }
// Execute executes a PQL query.
func (e *Executor) Execute(db string, q *pql.Query, slices []uint64) (interface{}, error) {
// Verify that a database is set.
if db == "" {
return nil, ErrDatabaseRequired
}
// If slices aren't specified, then include all of them.
if len(slices) == 0 {
// Round up the number of slices.
sliceN := (e.index.SliceN() % uint64(len(e.Cluster.Nodes))) + uint64(len(e.Cluster.Nodes))
// Generate a slices of all slices.
slices = make([]uint64, sliceN+1)
for i := range slices {
slices[i] = uint64(i)
}
}
return e.executeCall(db, q.Root, slices)
}
// executeCall executes a call.
func (e *Executor) executeCall(db string, c pql.Call, slices []uint64) (interface{}, error) {
switch c := c.(type) {
case pql.BitmapCall:
return e.executeBitmapCall(db, c, slices)
case *pql.Count:
return e.executeCount(db, c, slices)
case *pql.TopN:
return e.executeTopN(db, c, slices)
default:
panic("unreachable")
}
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint64) (*Bitmap, error) {
other := NewBitmap()
for node, nodeSlices := range e.slicesByNode(slices) {
// Execute locally if the hostname matches.
if node.Host == e.Host {
for _, slice := range nodeSlices {
bm, err := e.executeBitmapCallSlice(db, c, slice)
if err != nil {
return nil, err
}
other.Merge(bm)
}
continue
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices)
if err != nil {
return nil, err
}
other.Merge(res.(*Bitmap))
}
return other, nil
}
// executeBitmapCallSlice executes a bitmap call for a single slice.
func (e *Executor) executeBitmapCallSlice(db string, c pql.BitmapCall, slice uint64) (*Bitmap, error) {
switch c := c.(type) {
case *pql.Difference:
return e.executeDifferenceSlice(db, c, slice)
case *pql.Get:
return e.executeGetSlice(db, c, slice)
case *pql.Intersect:
return e.executeIntersectSlice(db, c, slice)
case *pql.Range:
return e.executeRangeSlice(db, c, slice)
case *pql.Union:
return e.executeUnionSlice(db, c, slice)
default:
panic("unreachable")
}
}
// executeTopN executes a top-n() call.
func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, error) {
panic("FIXME: calculate top n from each slice")
}
// executeDifferenceSlice executes a difference() call for a local slice.
func (e *Executor) executeDifferenceSlice(db string, c *pql.Difference, slice uint64) (*Bitmap, error) {
var other *Bitmap
for i, input := range c.Inputs {
bm, err := e.executeBitmapCallSlice(db, input, slice)
if err != nil {
return nil, err
}
if i == 0 {
other = bm
} else {
other = other.Difference(bm)
}
}
return other, nil
}
func (e *Executor) executeGetSlice(db string, c *pql.Get, slice uint64) (*Bitmap, error) {
frame := c.Frame
if frame == "" {
frame = DefaultFrame
}
f := e.Index().Fragment(db, frame, slice)
return f.Bitmap(c.ID), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
func (e *Executor) executeIntersectSlice(db string, c *pql.Intersect, slice uint64) (*Bitmap, error) {
var other *Bitmap
for i, input := range c.Inputs {
bm, err := e.executeBitmapCallSlice(db, input, slice)
if err != nil {
return nil, err
}
if i == 0 {
other = bm
} else {
other = other.Intersect(bm)
}
}
return other, nil
}
// executeRangeSlice executes a range() call for a local slice.
func (e *Executor) executeRangeSlice(db string, c *pql.Range, slice uint64) (*Bitmap, error) {
panic("FIXME")
}
// executeUnionSlice executes a union() call for a local slice.
func (e *Executor) executeUnionSlice(db string, c *pql.Union, slice uint64) (*Bitmap, error) {
var other *Bitmap
for i, input := range c.Inputs {
bm, err := e.executeBitmapCallSlice(db, input, slice)
if err != nil {
return nil, err
}
if i == 0 {
other = bm
} else {
other = other.Union(bm)
}
}
return other, nil
}
// executeCount executes a count() call.
func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint64, error) {
var n uint64
for node, nodeSlices := range e.slicesByNode(slices) {
// Execute locally if the hostname matches.
if node.Host == e.Host {
for _, slice := range nodeSlices {
bm, err := e.executeBitmapCallSlice(db, c.Input, slice)
if err != nil {
return 0, err
}
n += bm.Count()
}
continue
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices)
if err != nil {
return 0, err
}
n += res.(uint64)
}
return n, nil
}
// exec executes a PQL query remotely for a set of slices on a node.
func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64) (result interface{}, err error) {
// Encode request object.
buf, err := proto.Marshal(&internal.QueryRequest{
DB: proto.String(db),
Query: proto.String(q.String()),
Slices: slices,
})
if err != nil {
return nil, err
}
// Create HTTP request.
req, err := http.NewRequest("POST", (&url.URL{
Scheme: "http",
Host: node.Host,
Path: "/query",
}).String(), bytes.NewReader(buf))
if err != nil {
return nil, err
}
// Require protobuf encoding.
req.Header.Set("Accept", "application/x-protobuf")
req.Header.Set("Content-Type", "application/x-protobuf")
// Send request to remote node.
resp, err := e.HTTPClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
// Read response into buffer.
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
// Check status code.
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body)
}
// Decode response object.
var pb internal.QueryResponse
if err := proto.Unmarshal(body, &pb); err != nil {
return nil, err
}
// Return an error, if specified on response.
if err := decodeError(pb.GetErr()); err != nil {
return nil, err
}
// Return appropriate data for the query.
switch q.Root.(type) {
case pql.BitmapCall:
return decodeBitmap(pb.GetBitmap()), nil
case *pql.TopN:
return decodePairs(pb.GetPairs()), nil
case *pql.Count:
return pb.GetN(), nil
default:
panic(fmt.Sprintf("invalid node for remote exec: %T", q.Root))
}
}
// slicesByNode returns a mapping of nodes to slices.
//
// NOTE: Currently the only primary node is used.
func (e *Executor) slicesByNode(slices []uint64) map[*Node][]uint64 {
m := make(map[*Node][]uint64)
for _, slice := range slices {
nodes := e.Cluster.SliceNodes(slice)
node := nodes[0]
m[node] = append(m[node], slice)
}
return m
}
// decodeError returns an error representation of s if s is non-blank.
// Returns nil if s is blank.
func decodeError(s string) error {
if s == "" {
return nil
}
return errors.New(s)
}

800
executor/executor.go
View file

@ -1,800 +0,0 @@
package executor
import (
"encoding/gob"
"sort"
"time"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/query"
)
const DefaultTimeout = 30 * time.Second
type Executor struct {
inbox chan *db.Message
ID pilosa.GUID
Cluster *db.Cluster
ProcessMap *core.ProcessMap
PluginsPath string
Hold interface {
Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error)
Set(id *pilosa.GUID, value interface{}, timeout time.Duration)
}
Index interface {
ClearBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64) (bool, error)
Count(frag_id pilosa.SUUID, bitmap pilosa.BitmapHandle) (uint64, error)
Difference(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
FromBytes(frag_id pilosa.SUUID, bytes []byte) (pilosa.BitmapHandle, error)
Get(frag_id pilosa.SUUID, bitmap_id uint64) (pilosa.BitmapHandle, error)
GetBytes(frag_id pilosa.SUUID, bh pilosa.BitmapHandle) ([]byte, error)
Intersect(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
Range(frag_id pilosa.SUUID, bitmap_id uint64, start, end time.Time) (pilosa.BitmapHandle, error)
SetBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64, category uint64) (bool, error)
TopN(frag_id pilosa.SUUID, bh pilosa.BitmapHandle, n int, categories []uint64) ([]pilosa.Pair, error)
TopNAll(frag_id pilosa.SUUID, n int, categories []uint64) ([]pilosa.Pair, error)
Union(frag_id pilosa.SUUID, bh []pilosa.BitmapHandle) (pilosa.BitmapHandle, error)
}
TopologyMapper interface {
MakeFragments(db string, slice_int int) error
}
Transport interface {
Send(*db.Message, *pilosa.GUID)
}
}
func NewExecutor(id pilosa.GUID) *Executor {
log.Trace("NewExector")
return &Executor{inbox: make(chan *db.Message)}
}
func (self *Executor) Init() error {
log.Trace("Executor.Init()")
return nil
}
func (self *Executor) Close() {
log.Trace("Executor.Close()")
}
func (self *Executor) NewJob(job *db.Message) {
log.Trace("NewJob", job)
switch job.Data.(type) {
case query.CountQueryStep:
self.CountQueryStepHandler(job)
case query.TopNQueryStep:
self.TopNQueryStepHandler(job)
case query.UnionQueryStep:
self.UnionQueryStepHandler(job)
case query.IntersectQueryStep:
self.IntersectQueryStepHandler(job)
case query.DifferenceQueryStep:
self.DifferenceQueryStepHandler(job)
case query.CatQueryStep:
self.CatQueryStepHandler(job)
case query.GetQueryStep:
self.GetQueryStepHandler(job)
case query.SetQueryStep:
self.SetQueryStepHandler(job)
case query.ClearQueryStep:
self.ClearQueryStepHandler(job)
case query.RangeQueryStep:
self.RangeQueryStepHandler(job)
case query.StashQueryStep:
self.StashQueryStepHandler(job)
default:
log.Warn("unknown")
log.Warn(spew.Sdump(job.Data))
}
}
func (self *Executor) CountQueryStepHandler(msg *db.Message) {
log.Trace("CountQueryStepHandler")
//spew.Dump("COUNT QUERYSTEP")
qs := msg.Data.(query.CountQueryStep)
input := qs.Input
value, _ := self.Hold.Get(input, DefaultTimeout)
var bh pilosa.BitmapHandle
switch val := value.(type) {
case pilosa.BitmapHandle:
bh = val
case []byte:
bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val)
}
count, err := self.Index.Count(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
//spew.Dump("SLICE COUNT", count)
result_message := db.Message{
Data: query.CountQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: count},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) TopNQueryStepHandler(msg *db.Message) {
qs := msg.Data.(query.TopNQueryStep)
var bh pilosa.BitmapHandle
var topnPackage TopNPackage
// if we have an input, hold for it. if we don't, we assume an all() query
if qs.Input == nil {
topn, err := self.Index.TopNAll(qs.Location.FragmentId, qs.N*2, qs.Filters)
if err != nil {
log.Warn(spew.Sdump(err))
}
topnPackage = TopNPackage{*qs.Location.ProcessId, qs.Location.FragmentId, topn, bh}
} else {
input := qs.Input
value, _ := self.Hold.Get(input, 10)
//var bh pilosa.BitmapHandle
switch val := value.(type) {
case pilosa.BitmapHandle:
bh = val
case []byte:
bh, _ = self.Index.FromBytes(qs.Location.FragmentId, val)
}
topn, err := self.Index.TopN(qs.Location.FragmentId, bh, qs.N*2, qs.Filters)
if err != nil {
log.Warn(spew.Sdump(err))
}
topnPackage = TopNPackage{*qs.Location.ProcessId, qs.Location.FragmentId, topn, bh}
}
result_message := db.Message{
Data: query.TopNQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: topnPackage},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) UnionQueryStepHandler(msg *db.Message) {
log.Trace("UnionQueryStepHandler")
//spew.Dump("UNION QUERYSTEP")
qs := msg.Data.(query.UnionQueryStep)
var handles []pilosa.BitmapHandle
// create a list of bitmap handles
for _, input := range qs.Inputs {
value, _ := self.Hold.Get(input, DefaultTimeout)
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
handles = append(handles, bh)
}
}
bh, err := self.Index.Union(qs.Location.FragmentId, handles)
if err != nil {
spew.Dump(err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
result = bm
}
result_message := db.Message{
Data: query.UnionQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) IntersectQueryStepHandler(msg *db.Message) {
log.Trace("IntersectQueryStepHandler")
//spew.Dump("INTERSECT QUERYSTEP")
qs := msg.Data.(query.IntersectQueryStep)
var handles []pilosa.BitmapHandle
// create a list of bitmap handles
for _, input := range qs.Inputs {
value, _ := self.Hold.Get(input, DefaultTimeout)
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
handles = append(handles, bh)
}
}
bh, err := self.Index.Intersect(qs.Location.FragmentId, handles)
if err != nil {
spew.Dump(err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
result = bm
}
result_message := db.Message{
Data: query.IntersectQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) DifferenceQueryStepHandler(msg *db.Message) {
log.Trace("DifferenceQueryStepHandler")
//spew.Dump("DIFFERENCE QUERYSTEP")
qs := msg.Data.(query.DifferenceQueryStep)
var handles []pilosa.BitmapHandle
// create a list of bitmap handles
for _, input := range qs.Inputs {
value, _ := self.Hold.Get(input, DefaultTimeout)
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
handles = append(handles, bh)
}
}
bh, err := self.Index.Difference(qs.Location.FragmentId, handles)
if err != nil {
spew.Dump(err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
result = bm
}
result_message := db.Message{
Data: query.DifferenceQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) CatQueryStepHandler(msg *db.Message) {
log.Trace("CatQueryStepHandler")
qs := msg.Data.(query.CatQueryStep)
var handles []pilosa.BitmapHandle
return_type := "bitmap-handles"
var sum uint64
merge_map := make(map[uint64]uint64)
slice_map := make(map[uint64]map[pilosa.SUUID]struct{})
all_slice := make(map[pilosa.SUUID]struct {
process pilosa.GUID
handle pilosa.BitmapHandle
})
// either create a list of bitmap handles to cat (i.e. union), or sum the integer values
part := make(chan interface{})
num_parts := len(qs.Inputs)
for _, input := range qs.Inputs {
go func(id *pilosa.GUID, part chan interface{}) {
value, _ := self.Hold.Get(id, DefaultTimeout)
part <- value
}(input, part)
}
//for _, input := range qs.Inputs {
check_pair := false
for i := 0; i < num_parts; i++ {
value := <-part
switch val := value.(type) {
case pilosa.BitmapHandle:
handles = append(handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
handles = append(handles, bh)
case uint64:
//spew.Dump(val)
return_type = "sum"
sum += val
case TopNPackage:
return_type = "pair-list"
var e struct{}
for _, pair := range val.Pairs {
//merge_map[pair.Key] += pair.Count
if pair.Key == 0 {
continue //skip
}
merge_map[pair.Key] += pair.Count
mm, ok := slice_map[pair.Key]
if !ok {
mm = make(map[pilosa.SUUID]struct{})
slice_map[pair.Key] = mm
}
mm[val.FragmentId] = e
}
all_slice[val.FragmentId] = struct {
process pilosa.GUID
handle pilosa.BitmapHandle
}{val.ProcessId, val.HBitmap}
check_pair = true
}
}
if check_pair { //no point in doing this for non top-n handling
tasks := BuildTask(merge_map, slice_map, all_slice)
self.FetchMissing(tasks)
for k, v := range self.GatherResults(tasks) {
merge_map[k] += v
}
}
// either return the sum, or return the compressed bitmap resulting from the cat (union)
var result interface{}
if return_type == "sum" {
result = sum
} else if return_type == "bitmap-handles" {
bh, err := self.Index.Union(qs.Location.FragmentId, handles)
result, err = self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
} else if return_type == "pair-list" {
rank_list := make(pilosa.RankList, 0, len(merge_map))
for k, v := range merge_map {
if k == 0 || v == 0 {
continue //shouldn't be getting 0 keys or values anyway
}
rank := new(pilosa.Rank)
rank.Pair = &pilosa.Pair{Key: k, Count: v}
rank_list = append(rank_list, rank)
}
sort.Sort(rank_list) // kinda seems like this copy is wasteful..i'll ponder
items_size := min(len(merge_map), qs.N)
pair_list := make([]pilosa.Pair, 0, items_size+1)
for i, r := range rank_list {
if i < items_size {
pair_list = append(pair_list, *r.Pair)
} else {
break
}
}
result = pair_list
} else {
result = "NONE"
}
result_message := db.Message{
Data: query.CatQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) SendRequest(process_id pilosa.GUID, t *Task) {
args := make([]pilosa.FillArgs, len(t.f), len(t.f))
for _, v := range t.f {
args = append(args, v)
}
msg := new(db.Message)
p, _ := self.ProcessMap.GetProcess(&self.ID)
msg.Data = TopFill{args, p.Id(), t.hold_id, process_id}
self.Transport.Send(msg, &process_id)
}
func (self *Executor) FetchMissing(tasks map[pilosa.GUID]*Task) {
for k, v := range tasks {
go self.SendRequest(k, v)
}
}
func (self *Executor) GatherResults(tasks map[pilosa.GUID]*Task) map[uint64]uint64 {
results := make(map[uint64]uint64)
answers := make(chan []pilosa.Pair)
for _, task := range tasks {
go func(id pilosa.GUID) {
value, err := self.Hold.Get(&id, 10) //eiher need to be the frame process or the handler process?
if value == nil {
log.Warn("Bad TopN Result:", err)
empty := make([]pilosa.Pair, 0, 0)
answers <- empty
} else {
answers <- value.([]pilosa.Pair)
}
}(task.hold_id)
}
for i := 0; i < len(tasks); i++ {
batch := <-answers
for _, pair := range batch {
results[pair.Key] += pair.Count
}
}
close(answers)
return results
}
func (self *Executor) GetQueryStepHandler(msg *db.Message) {
qs := msg.Data.(query.GetQueryStep)
//spew.Dump("GET QUERYSTEP")
bh, err := self.Index.Get(qs.Location.FragmentId, qs.Bitmap.Id)
if err != nil {
spew.Dump(err)
log.Error("GetQueryStepHandler1", qs.Location.FragmentId.String(), qs.Bitmap.Id)
log.Error("GetQueryStepHandler2", err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
log.Error("GetQueryStepHandlerr3", qs.Location.FragmentId.String(), qs.Bitmap.Id)
log.Error("GetQueryStepHandler4", err)
}
result = bm
}
result_message := db.Message{
Data: query.GetQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) SetQueryStepHandler(msg *db.Message) {
//spew.Dump("SET QUERYSTEP")
qs := msg.Data.(query.SetQueryStep)
result, _ := self.Index.SetBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId, qs.Bitmap.Filter)
result_message := db.Message{
Data: query.SetQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) ClearQueryStepHandler(msg *db.Message) {
//spew.Dump("SET QUERYSTEP")
qs := msg.Data.(query.ClearQueryStep)
result, _ := self.Index.ClearBit(qs.Location.FragmentId, qs.Bitmap.Id, qs.ProfileId)
result_message := db.Message{
Data: query.ClearQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) RangeQueryStepHandler(msg *db.Message) {
qs := msg.Data.(query.RangeQueryStep)
//spew.Dump("RANDE QUERYSTEP")
bh, err := self.Index.Range(qs.Location.FragmentId, qs.Bitmap.Id, qs.Start, qs.End)
if err != nil {
spew.Dump(err)
}
var result interface{}
if qs.LocIsDest() {
result = bh
} else {
bm, err := self.Index.GetBytes(qs.Location.FragmentId, bh)
if err != nil {
spew.Dump(err)
}
result = bm
}
result_message := db.Message{
Data: query.RangeQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) StashQueryStepHandler(msg *db.Message) {
log.Trace("StashQueryStepHandler")
qs := msg.Data.(query.StashQueryStep)
part := make(chan interface{})
num_parts := len(qs.Inputs)
for _, input := range qs.Inputs {
go func(id *pilosa.GUID, part chan interface{}) {
value, _ := self.Hold.Get(id, DefaultTimeout)
part <- value
}(input, part)
}
//just collect all the handles and return them
result := query.NewStash() //query.Stash{make([]query.CacheItem, 0), false}
for i := 0; i < num_parts; i++ {
value := <-part
switch val := value.(type) {
case pilosa.BitmapHandle:
log.Info("STASH ADDING HANDLE", val)
//not sure what to do here....
//result.Handles = append(result.Handles, val)
case []byte:
bh, _ := self.Index.FromBytes(qs.Location.FragmentId, val)
item := query.CacheItem{FragmentId: qs.Location.FragmentId, Handle: bh}
result.Stash = append(result.Stash, item)
case query.Stash:
result.Stash = append(result.Stash, val.Stash...)
default:
log.Warn("UNEXCPECTED MESSAGE", value)
}
}
result_message := db.Message{
Data: query.StashQueryResult{
BaseQueryResult: &query.BaseQueryResult{Id: qs.Id, Data: result},
},
}
self.Transport.Send(&result_message, qs.Destination.ProcessId)
}
func (self *Executor) RunQueryTest(database_name string, pql string) string {
return pql
}
func (self *Executor) runQuery(database *db.Database, qry *query.Query) error {
log.Trace("Executor.runQuery", database, qry)
process, err := self.ProcessMap.GetProcess(&self.ID)
if err != nil {
return err
}
process_id := process.Id()
fragment_id := pilosa.SUUID(0)
destination := db.Location{ProcessId: &process_id, FragmentId: fragment_id}
query_plan, err := query.QueryPlanForQuery(database, qry, &destination)
if err != nil {
switch obj := err.(type) {
case *query.FragmentNotFound:
self.TopologyMapper.MakeFragments(obj.Db, obj.Slice)
}
self.Hold.Set(qry.Id, err, 30)
return err
}
// loop over the query steps and send to Transport
for _, qs := range *query_plan {
msg := new(db.Message)
msg.Data = qs
switch step := qs.(type) {
case query.PortableQueryStep:
loc := step.GetLocation()
if loc != nil {
self.Transport.Send(msg, loc.ProcessId)
} else {
log.Warn("Problem with querystep(nil location)", spew.Sdump(step))
}
}
}
return nil
}
func (self *Executor) RunPQL(database_name string, pql string) (interface{}, error) {
log.Trace("Executor.RunPQL", database_name, pql)
database := self.Cluster.GetOrCreateDatabase(database_name)
// see if the outer query function is a custom query
reserved_functions := stringSlice{"get", "set", "clear", "union", "intersect", "difference", "count", "top-n", "mask", "range", "stash", "recall"}
tokens, err := query.Lex(pql)
if err != nil {
return nil, err
}
outer_token := tokens[0].Text
if reserved_functions.pos(outer_token) != -1 {
qry, err := query.QueryForTokens(tokens)
if err != nil {
return nil, err
}
go self.runQuery(database, qry)
var final interface{}
final, err = self.Hold.Get(qry.Id, 10)
if err != nil {
return nil, err
}
return final, nil
} else { //want to refactor this down to just RunPlugin(tokens)
plugins_file := self.PluginsPath + "/" + outer_token + ".js"
filter, filters := query.TokensToFilterStrings(tokens)
query_list := GetPlugin(plugins_file, filter, filters).(query.PqlList)
for i, _ := range query_list {
qry, err := query.QueryForPQL(query_list[i].PQL)
if err != nil {
return nil, err
}
go self.runQuery(database, qry)
query_list[i].Id = qry.Id
}
final_result := make(map[string]interface{})
result := make(chan struct {
final interface{}
label string
err error
})
x := 0
for i, _ := range query_list {
x++
go func(q query.PqlListItem, reply chan struct {
final interface{}
label string
err error
}) {
final, err := self.Hold.Get(q.Id, 10)
result <- struct {
final interface{}
label string
err error
}{final, q.Label, err}
}(query_list[i], result)
if err != nil {
out := spew.Sdump(err)
log.Warn(out)
}
}
for z := 0; z < x; z++ {
ans := <-result
final_result[ans.label] = ans.final
}
return final_result, nil
}
}
func init() {
gob.Register(TopNPackage{})
gob.Register(TopFill{})
}
type TopNPackage struct {
ProcessId pilosa.GUID
FragmentId pilosa.SUUID
Pairs []pilosa.Pair
HBitmap pilosa.BitmapHandle
}
type TopFill struct {
Args []pilosa.FillArgs
ReturnProcessId pilosa.GUID
QueryId pilosa.GUID
DestProcessId pilosa.GUID
}
type Task struct {
processid pilosa.GUID
f map[pilosa.SUUID]pilosa.FillArgs
hold_id pilosa.GUID
}
func newtask(p pilosa.GUID) *Task {
result := new(Task)
result.processid = p
result.f = make(map[pilosa.SUUID]pilosa.FillArgs)
result.hold_id = pilosa.NewGUID()
return result
}
func (t *Task) Add(frag pilosa.SUUID, bitmap_id uint64, handle pilosa.BitmapHandle) {
fa, ok := t.f[frag]
if !ok {
fa = pilosa.FillArgs{Frag_id: frag, Handle: handle, Bitmaps: make([]uint64, 0, 0)}
}
fa.Bitmaps = append(fa.Bitmaps, bitmap_id)
t.f[frag] = fa
}
func BuildTask(merge_map map[uint64]uint64,
slice_map map[uint64]map[pilosa.SUUID]struct{},
total_fragments map[pilosa.SUUID]struct {
process pilosa.GUID
handle pilosa.BitmapHandle
}) map[pilosa.GUID]*Task {
tasks := make(map[pilosa.GUID]*Task)
for bitmap_id, _ := range merge_map { //for all brands
//for fragment_id, reported_fragments := range slice_map[bitmap_id] { //find missing fragments
reporting_fragments := slice_map[bitmap_id]
//id slice ==> SUUID,BitmapHandle
for _, p := range missing(reporting_fragments, total_fragments) {
task, ok := tasks[p.process]
if !ok {
task = newtask(p.process)
tasks[p.process] = task
}
task.Add(p.fragment, bitmap_id, p.handle)
}
//}
}
return tasks
}
type hole struct {
process pilosa.GUID
handle pilosa.BitmapHandle
fragment pilosa.SUUID
}
func missing(fids map[pilosa.SUUID]struct{}, all map[pilosa.SUUID]struct {
process pilosa.GUID
handle pilosa.BitmapHandle
}) []hole {
results := make([]hole, 0, 0)
for k, v := range all {
_, ok := fids[k]
if !ok {
results = append(results, hole{v.process, v.handle, k})
}
}
return results
}
func (self *TopFill) GetId() *pilosa.GUID {
return &self.QueryId
}
func (self *TopFill) GetLocation() *db.Location {
return &db.Location{ProcessId: &self.DestProcessId, FragmentId: 0} //this message is a broadcast to many fragments so i'm choosing fragmentzero
}
func (self *Executor) Run() {
log.Warn("Executor Run...")
}
type stringSlice []string
func (slice stringSlice) pos(value string) int {
for p, v := range slice {
if v == value {
return p
}
}
return -1
}
func min(a, b int) int {
if a < b {
return a
}
return b
}

57
executor/utils.go
View file

@ -1,57 +0,0 @@
package executor
import (
"bytes"
"io/ioutil"
"strings"
"github.com/davecgh/go-spew/spew"
"github.com/robertkrimen/otto"
"github.com/umbel/pilosa/query"
)
func GetPlugin(file_name string, filter string, filters []string) interface{} {
file_data, err := ioutil.ReadFile(file_name)
if err != nil {
spew.Dump(err)
}
s := string(file_data[:])
// convert the list of filters in to a string array
var buffer bytes.Buffer
if len(filters) > 0 {
buffer.WriteString("['")
buffer.WriteString(strings.Join(filters, "','"))
buffer.WriteString("']")
} else {
buffer.WriteString("[]")
}
js := "query_list = (" + s + ")('" + filter + "', " + buffer.String() + ");"
Otto := otto.New()
Otto.Run(js)
query_objects, err := Otto.Get("query_list")
query_list_interface, err := query_objects.Export()
if err != nil {
spew.Dump(err)
}
var query_list query.PqlList
// ql is []interface{}
switch ql := query_list_interface.(type) {
case []interface{}:
// q is map[string]interface{}
for i, _ := range ql {
q := ql[i].(map[string]interface{})
query_list = append(query_list, query.PqlListItem{Label: q["label"].(string), PQL: q["pql"].(string)})
}
default:
spew.Dump("DEFAULT")
}
return query_list
}

198
executor_test.go Normal file
View file

@ -0,0 +1,198 @@
package pilosa_test
import (
"reflect"
"strings"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/pql"
)
// Ensure a get query can be executed.
func TestExecutor_Execute_Get(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "f", 0).Bitmap(10).SetBit(3)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit(SliceWidth + 1)
if res, err := e.Execute("d", MustParse(`get(id=10, frame=f)`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 8 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 2 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
}
}
// Ensure a difference query can be executed.
func TestExecutor_Execute_Difference(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(1)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(2)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(3)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(2)
if res, err := e.Execute("d", MustParse(`difference(get(id=10), get(id=11))`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 1 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 10 { // b1010
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
}
}
// Ensure an intersect query can be executed.
func TestExecutor_Execute_Intersect(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(1)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(SliceWidth + 1)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(SliceWidth + 2)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(1)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(2)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(SliceWidth + 2)
if res, err := e.Execute("d", MustParse(`intersect(get(id=10), get(id=11))`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 2 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 4 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
}
}
// Ensure a union query can be executed.
func TestExecutor_Execute_Union(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(0)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(SliceWidth + 1)
e.Index().Fragment("d", "general", 0).Bitmap(10).SetBit(SliceWidth + 2)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(2)
e.Index().Fragment("d", "general", 0).Bitmap(11).SetBit(SliceWidth + 2)
if res, err := e.Execute("d", MustParse(`union(get(id=10), get(id=11))`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 5 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 6 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
}
}
// Ensure a count query can be executed.
func TestExecutor_Execute_Count(t *testing.T) {
e := NewExecutor(NewCluster(1))
e.Index().Fragment("d", "f", 0).Bitmap(10).SetBit(3)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit(SliceWidth + 1)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit(SliceWidth + 2)
if n, err := e.Execute("d", MustParse(`count(get(id=10, frame=f))`), nil); err != nil {
t.Fatal(err)
} else if n != uint64(3) {
t.Fatalf("unexpected n: %d", n)
}
}
// Ensure a remote query can return a bitmap.
func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c := NewCluster(2)
// Create secondary server and update second cluster node.
s := NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `get(id=10, frame=f)` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 2}) {
t.Fatalf("unexpected slices: %+v", slices)
}
// Set bits in slice 0 & 2.
bm := pilosa.NewBitmap()
bm.SetBit((0 * SliceWidth) + 1)
bm.SetBit((0 * SliceWidth) + 2)
bm.SetBit((2 * SliceWidth) + 4)
return bm, nil
}
// Create local executor data.
// The local node owns slice 1.
e := NewExecutor(c)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit((1 * SliceWidth) + 1)
if res, err := e.Execute("d", MustParse(`get(id=10, frame=f)`), nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 3 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
} else if chunks[0].Value[0] != 6 {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 2 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
}
}
// Ensure a remote query can return a count.
func TestExecutor_Execute_Remote_Count(t *testing.T) {
c := NewCluster(2)
// Create secondary server and update second cluster node.
s := NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return uint64(10), nil
}
// Create local executor data. The local node owns slice 1.
e := NewExecutor(c)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit((1 * SliceWidth) + 1)
e.Index().Fragment("d", "f", 1).Bitmap(10).SetBit((1 * SliceWidth) + 2)
if n, err := e.Execute("d", MustParse(`count(get(id=10, frame=f))`), nil); err != nil {
t.Fatal(err)
} else if n != uint64(12) {
t.Fatalf("unexpected n: %d", n)
}
}
// Executor represents a test wrapper for pilosa.Executor.
type Executor struct {
*pilosa.Executor
}
// NewExecutor returns a new instance of Executor.
// The executor always matches the hostname of the first cluster node.
func NewExecutor(cluster *pilosa.Cluster) *Executor {
e := &Executor{
Executor: pilosa.NewExecutor(pilosa.NewIndex()),
}
e.Cluster = cluster
e.Host = cluster.Nodes[0].Host
return e
}
// MustParse parses s into a PQL query. Panic on error.
func MustParse(s string) *pql.Query {
q, err := pql.NewParser(strings.NewReader(s)).Parse()
if err != nil {
panic(err)
}
return q
}

329
fragment.go
View file

@ -1,213 +1,230 @@
package pilosa
import (
"sort"
"strings"
"sync"
"time"
log "github.com/cihub/seelog"
"github.com/golang/groupcache/lru"
)
// SliceWidth is the number of profile IDs in a slice.
const SliceWidth = 65536
// Fragment represents the intersection of a frame and slice in a database.
type Fragment struct {
mu sync.Mutex
id SUUID
slice int
impl Pilosa
cache *lru.Cache
mu sync.Mutex
// Provide an autoincrementing index for bitmap handles.
seq uint64
// Composite identifiers
db string
frame string
slice uint64
// Stats for how many messages have been processed.
stats FragmentStats
// Bitmap cache.
cache Cache
}
func NewFragment(id SUUID, db string, slice int, frame string) *Fragment {
storage := NewStorage(Backend, StorageOptions{
DB: db,
Slice: slice,
Frame: frame,
FragmentID: id,
LevelDBPath: LevelDBPath,
})
var impl Pilosa
if strings.HasSuffix(frame, ".n") {
impl = NewBrand(db, frame, slice, storage, 50000, 45000, 100)
} else {
impl = NewGeneral(db, frame, slice, storage)
}
return &Fragment{
id: id,
cache: lru.New(50000),
impl: impl,
// NewFragment returns a new instance of Fragment.
func NewFragment(db, frame string, slice uint64) *Fragment {
f := &Fragment{
db: db,
frame: frame,
slice: slice,
}
}
func (f *Fragment) Bitmap(bh BitmapHandle) (*Bitmap, bool) {
f.mu.Lock()
defer f.mu.Unlock()
return f.bitmap(bh)
}
func (f *Fragment) bitmap(bh BitmapHandle) (*Bitmap, bool) {
bm, ok := f.cache.Get(bh)
if ok && bm != nil {
return bm.(*Bitmap), ok
// Determine cache type from frame name.
if strings.HasSuffix(frame, ".n") {
c := NewRankCache()
c.ThresholdLength = 50000
c.ThresholdIndex = 45000
f.cache = c
} else {
f.cache = NewLRUCache(50000)
}
return NewBitmap(), false //cache fail
return f
}
func (f *Fragment) exists(bitmapID uint64) bool {
// Bitmap returns a bitmap by ID.
func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap {
f.mu.Lock()
defer f.mu.Unlock()
return f.impl.Exists(bitmapID)
return f.bitmap(bitmapID)
}
func (f *Fragment) bitmap(bitmapID uint64) *Bitmap {
// Read from cache.
if bm, ok := f.cache.Get(bitmapID); ok {
return bm
}
// Read from storage engine.
// bm, filter := f.storage.Fetch(bitmapID, f.db, f.frame, f.slice)
bm := NewBitmap()
f.cache.Add(bitmapID, 0 /*filter*/, bm)
return bm
}
func (f *Fragment) TopNAll(n int, categories []uint64) []Pair {
f.mu.Lock()
defer f.mu.Unlock()
return f.impl.TopNAll(n, categories)
}
f.cache.Invalidate()
func (f *Fragment) TopN(bitmap BitmapHandle, n int, categories []uint64) []Pair {
f.mu.Lock()
defer f.mu.Unlock()
bm, ok := f.cache.Get(bitmap)
if ok {
return f.impl.TopN(bm.(*Bitmap), n, categories)
// Create a set of categories.
m := make(map[uint64]struct{})
for _, v := range categories {
m[v] = struct{}{}
}
return nil
}
func (f *Fragment) NewHandle(bitmapID uint64) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
return f.allocHandle(f.impl.Get(bitmapID))
}
// Iterate over rankings and add to results until we have enough.
var results []Pair
for _, pair := range f.cache.Pairs() {
// Skip if categories are specified but category is not found.
if _, ok := m[pair.category]; (len(categories) > 0 && !ok) || pair.Count <= 0 {
continue
}
func (f *Fragment) AllocHandle(bm *Bitmap) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
return f.allocHandle(bm)
}
// Append pair.
results = append(results, pair)
func (f *Fragment) allocHandle(bm *Bitmap) BitmapHandle {
handle := f.nextHandle()
f.cache.Add(handle, bm)
return handle
}
func (f *Fragment) nextHandle() BitmapHandle {
millis := uint64(time.Now().UTC().UnixNano())
id := millis << (64 - 41)
id |= uint64(f.slice) << (64 - 41 - 13)
id |= f.seq % 1024
f.seq += 1
return BitmapHandle(id)
}
func (f *Fragment) Union(bitmaps []BitmapHandle) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
result := NewBitmap()
for i, id := range bitmaps {
bm, _ := f.bitmap(id)
if i == 0 {
result = bm
} else {
result = result.Union(bm)
// Exit when we have enough pairs.
if len(results) >= n {
break
}
}
return f.allocHandle(result)
return results
}
func (f *Fragment) build_time_range_bitmap(bitmapID uint64, start, end time.Time) BitmapHandle {
func (f *Fragment) TopN(src *Bitmap, n int, categories []uint64) []Pair {
f.mu.Lock()
defer f.mu.Unlock()
result := NewBitmap()
for i, bid := range GetRange(start, end, bitmapID) {
bm := f.impl.Get(bid)
if i == 0 {
result = bm
} else {
result = result.Union(bm)
// Resort rank, if necessary.
f.cache.Invalidate()
// Create a set of categories.
set := make(map[uint64]struct{})
for _, v := range categories {
set[v] = struct{}{}
}
var results []Pair
var x int
breakout := 1000
// Iterate over rankings.
rankings := f.cache.Pairs()
for i, pair := range rankings {
// Skip if category not found.
if len(set) > 0 {
if _, ok := set[pair.category]; !ok {
continue
}
}
// Only append if there are intersecting bits with source bitmap.
bc := src.IntersectionCount(pair.bitmap)
if bc > 0 {
results = append(results, Pair{
Key: pair.Key,
Count: bc,
category: pair.category,
})
}
x = i
// Exit when we have enough.
if len(results) > n {
break
}
}
return f.AllocHandle(result)
}
func (f *Fragment) Intersect(bitmaps []BitmapHandle) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
// Sort results by ranking.
sort.Sort(Pairs(results))
var result *Bitmap
for i, id := range bitmaps {
bm, _ := f.bitmap(id)
if i == 0 {
result = bm.Clone()
} else {
result = result.Intersection(bm)
if len(results) < n {
return results
}
end := len(results) - 1
o := results[end]
threshold := o.Count
if threshold <= 10 {
return results
}
results = append(results, o)
for i := x + 1; i < len(rankings); i++ {
o = rankings[i]
if len(set) > 0 {
if _, ok := set[o.category]; !ok {
continue
}
}
// Need something to do with the size of initial bitmap
if len(results) > breakout || o.Count < threshold {
break
}
bc := src.IntersectionCount(o.bitmap)
if bc > threshold {
if results[end-1].Count > bc {
results[end] = Pair{Key: o.Key, Count: bc, category: o.category}
threshold = bc
} else {
results[end+1] = Pair{Key: o.Key, Count: bc, category: o.category}
sort.Sort(Pairs(results))
threshold = results[end].Count
}
}
}
return f.allocHandle(result)
return results[:end]
}
func (f *Fragment) Difference(bitmaps []BitmapHandle) BitmapHandle {
f.mu.Lock()
defer f.mu.Unlock()
/*
func (f *Fragment) TopFill(args FillArgs) ([]Pair, error) {
result := make([]Pair, 0)
for _, id := range args.Bitmaps {
if _, ok := f.cache.Get(id); !ok {
continue
}
result := NewBitmap()
for i, id := range bitmaps {
bm, _ := f.bitmap(id)
if i == 0 {
result = bm
} else {
result = result.Difference(bm)
if args.Handle == 0 {
if bm := f.Bitmap(id); bm != nil && bm.Count() > 0 {
result = append(result, Pair{Key: id, Count: bm.Count()})
}
continue
}
res := f.Intersect([]uint64{args.Handle, id})
if res == nil {
continue
}
if bc := res.BitCount(); bc > 0 {
result = append(result, Pair{Key: id, Count: bc})
}
}
return f.allocHandle(result)
return result, nil
}
*/
func (f *Fragment) Persist() {
func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap {
f.mu.Lock()
defer f.mu.Unlock()
err := f.impl.Persist()
if err != nil {
log.Warn("Error saving:", err)
bitmapIDs := GetRange(start, end, bitmapID)
if len(bitmapIDs) == 0 {
return NewBitmap()
}
}
func (f *Fragment) Load() {
f.mu.Lock()
defer f.mu.Unlock()
f.impl.Load(f)
}
// FragmentStats represents in-memory stats for a single fragment.
type FragmentStats struct {
// Messages processed by the fragment
ProcessN uint64
ProcessTime time.Duration
}
type Pilosa interface {
Get(id uint64) *Bitmap
SetBit(id uint64, bit_pos uint64, filter uint64) bool
ClearBit(id uint64, bit_pos uint64) bool
TopN(b *Bitmap, n int, categories []uint64) []Pair
TopNAll(n int, categories []uint64) []Pair
Clear() bool
Store(bitmapID uint64, bm *Bitmap, filter uint64) error
Stats() interface{}
Persist() error
Load(fragment *Fragment)
Exists(id uint64) bool
result := f.bitmap(bitmapIDs[0])
for _, id := range bitmapIDs[1:] {
result = result.Union(f.bitmap(id))
}
return result
}

340
fragment_container.go
View file

@ -1,340 +0,0 @@
package pilosa
import (
"bytes"
"compress/gzip"
"encoding/base64"
"encoding/gob"
"errors"
"fmt"
"io/ioutil"
"sync"
"syscall"
"time"
log "github.com/cihub/seelog"
// "github.com/umbel/pilosa/statsd"
)
// DefaultBackend is the default data storage layer.
const DefaultBackend = "cassandra"
var Backend = DefaultBackend
var LevelDBPath string
var (
ErrFragmentNotFound = errors.New("fragment not found")
)
func init() {
gob.Register(BitmapHandle(0))
gob.Register([]Pair{})
}
type FragmentContainer struct {
mu sync.Mutex
fragments map[SUUID]*Fragment
}
func NewFragmentContainer() *FragmentContainer {
return &FragmentContainer{
fragments: make(map[SUUID]*Fragment),
}
}
type BitmapHandle uint64
type FillArgs struct {
FragmentID SUUID
Handle BitmapHandle
Bitmaps []uint64
}
func (fc *FragmentContainer) Fragment(fragmentID SUUID) (*Fragment, bool) {
fc.mu.Lock()
f, ok := fc.fragments[fragmentID]
fc.mu.Unlock()
return f, ok
}
func (fc *FragmentContainer) Get(fragmentID SUUID, bitmapID uint64) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.NewHandle(bitmapID), nil
}
func (fc *FragmentContainer) LoadBitmap(fragmentID SUUID, bitmapID uint64, data string, filter uint64) error {
f, ok := fc.Fragment(fragmentID)
if !ok {
return ErrFragmentNotFound
}
// Decode from base64 encoding.
buf, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return err
}
// Decompress data.
reader, err := gzip.NewReader(bytes.NewReader(buf))
if err != nil {
return err
}
b, err := ioutil.ReadAll(reader)
if err != nil {
return err
}
// Build bitmap from data.
bm := NewBitmap()
bm.FromBytes(b)
// Write bitmap to the underlying store.
return f.impl.Store(bitmapID, bm, filter)
}
func (fc *FragmentContainer) Stats(fragmentID SUUID) interface{} {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil
}
return f.impl.Stats()
}
func (fc *FragmentContainer) Empty(fragmentID SUUID) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.AllocHandle(NewBitmap()), nil
}
func (fc *FragmentContainer) Intersect(fragmentID SUUID, bh []BitmapHandle) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.Intersect(bh), nil
}
func (fc *FragmentContainer) Union(fragmentID SUUID, bh []BitmapHandle) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.Union(bh), nil
}
func (fc *FragmentContainer) Difference(fragmentID SUUID, bh []BitmapHandle) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.Difference(bh), nil
}
func (fc *FragmentContainer) Mask(fragmentID SUUID, start, end uint64) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
bm := NewBitmap()
for i := start; i < end; i++ {
bm.SetBit(i)
}
return f.AllocHandle(bm), nil
}
func (fc *FragmentContainer) Range(fragmentID SUUID, bitmapID uint64, start, end time.Time) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
return f.build_time_range_bitmap(bitmapID, start, end), nil
}
func (fc *FragmentContainer) TopN(fragmentID SUUID, bh BitmapHandle, n int, categories []uint64) ([]Pair, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
return f.TopN(bh, n, categories), nil
}
func (fc *FragmentContainer) TopNAll(fragmentID SUUID, n int, categories []uint64) ([]Pair, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
return f.TopNAll(n, categories), nil
}
func (fc *FragmentContainer) TopFillBatch(args []FillArgs) ([]Pair, error) {
results := make(map[uint64]uint64)
for _, v := range args {
items, _ := fc.TopFillFragment(v)
if len(args) == 1 {
return items, nil
}
for _, pair := range items {
results[pair.Key] += pair.Count
}
}
ret_val := make([]Pair, len(results))
for k, v := range results {
if v > 0 { //don't include 0 size items
ret_val = append(ret_val, Pair{k, v})
}
}
return ret_val, nil
}
func (fc *FragmentContainer) TopFillFragment(args FillArgs) ([]Pair, error) {
f, ok := fc.Fragment(args.FragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
result := make([]Pair, 0)
for _, v := range args.Bitmaps {
if !f.exists(v) {
continue
}
a := f.NewHandle(v)
if args.Handle == 0 {
// Return just the count
if bm, ok := f.Bitmap(a); ok && bm.Count() > 0 {
result = append(result, Pair{v, bm.Count()})
}
continue
}
res := f.Intersect([]BitmapHandle{args.Handle, a})
if bm, ok := f.Bitmap(res); ok {
bc := bm.BitCount()
if bc > 0 {
result = append(result, Pair{v, bc})
}
}
}
return result, nil
}
func (fc *FragmentContainer) GetList(fragmentID SUUID, bitmapIDs []uint64) ([]BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
a := make([]BitmapHandle, len(bitmapIDs))
for i, v := range bitmapIDs {
a[i] = f.NewHandle(v)
}
return a, nil
}
func (fc *FragmentContainer) Count(fragmentID SUUID, bh BitmapHandle) (uint64, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
bm, ok := f.Bitmap(bh)
if ok == false {
return 0, nil
}
return bm.BitCount(), nil
}
func (fc *FragmentContainer) GetBytes(fragmentID SUUID, bh BitmapHandle) ([]byte, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return nil, ErrFragmentNotFound
}
bm, ok := f.Bitmap(bh)
if !ok {
bm = NewBitmap()
log.Warn("cache miss")
}
var b bytes.Buffer
w := gzip.NewWriter(&b)
if _, err := w.Write(bm.ToBytes()); err != nil {
return nil, err
}
if err := w.Flush(); err != nil {
return nil, err
}
if err := w.Close(); err != nil {
return nil, err
}
return b.Bytes(), nil
}
func (fc *FragmentContainer) FromBytes(fragmentID SUUID, data []byte) (BitmapHandle, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return 0, ErrFragmentNotFound
}
r, _ := gzip.NewReader(bytes.NewReader(data))
b, _ := ioutil.ReadAll(r)
bm := NewBitmap()
bm.FromBytes(b)
return f.AllocHandle(bm), nil
}
func (fc *FragmentContainer) SetBit(fragmentID SUUID, bitmapID uint64, pos uint64, category uint64) (bool, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return false, ErrFragmentNotFound
}
return f.impl.SetBit(bitmapID, pos, category), nil
}
func (fc *FragmentContainer) ClearBit(fragmentID SUUID, bitmapID uint64, pos uint64) (bool, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return false, ErrFragmentNotFound
}
return f.impl.ClearBit(bitmapID, pos), nil
}
func (fc *FragmentContainer) Clear(fragmentID SUUID) (bool, error) {
f, ok := fc.Fragment(fragmentID)
if !ok {
return false, ErrFragmentNotFound
}
return f.impl.Clear(), nil
}
func (fc *FragmentContainer) AddFragment(db string, frame string, slice int, id SUUID) {
fc.mu.Lock()
defer fc.mu.Unlock()
_, ok := fc.fragments[id]
if ok {
return
}
log.Warn("ADD FRAGMENT", frame, db, slice, id.String())
f := NewFragment(id, db, slice, frame)
fc.fragments[id] = f
// go f.Load(loader)
}
func dumpHandlesToLog() {
var limit syscall.Rlimit
if err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &limit); err != nil {
log.Warn("Getrlimit:" + err.Error())
}
mesg := fmt.Sprintf("%v file descriptors out of a maximum of %v available\n", limit.Cur, limit.Max)
log.Warn(mesg)
}

308
fragment_container_test.go
View file

@ -1,308 +0,0 @@
package pilosa_test
import (
"testing"
"github.com/umbel/pilosa"
_ "github.com/umbel/pilosa/storage"
)
func init() {
pilosa.Backend = "memory"
}
// Ensure a fragment can be retrieved from the container.
func TestFragmentContainer_Get(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustClear(1)
if bh, err := fc.Get(pilosa.SUUID(1), 1234); err != nil {
t.Fatal(err)
} else if bh == 0 {
t.Fatal("expected non-zero bitmap handle")
}
}
// Ensure a bit can be set on a bitmap.
func TestFragmentContainer_SetBit(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustClear(1)
// Set a bit on the bitmap.
if changed, err := fc.SetBit(1, uint64(1234), 1, 0); err != nil {
t.Fatal(err)
} else if changed == false {
t.Fatal("expected change")
}
// Set the same bit on the bitmap. No change should be indicated.
if changed, err := fc.SetBit(1, uint64(1234), 1, 0); err != nil {
t.Fatal(err)
} else if changed == true {
t.Fatal("expected no change")
}
}
// Ensure the number of bits on a bitmap can be counted.
func TestFragmentContainer_Count(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, pilosa.SUUID(1))
// Set a bit on the bitmap.
bi1 := uint64(1234)
if changed, err := fc.SetBit(pilosa.SUUID(1), bi1, 1, 0); err != nil {
t.Fatal(err)
} else if changed == false {
t.Fatal("expected change")
}
// Verify that one bit is set.
if bh, err := fc.Get(pilosa.SUUID(1), bi1); err != nil {
t.Fatal(err)
} else if n, err := fc.Count(pilosa.SUUID(1), bh); err != nil {
t.Fatal(err)
} else if n != 1 {
t.Fatalf("unexpected count: %d", n)
}
}
// Ensure the bits in two bitmaps can be unioned.
func TestFragmentContainer_Union(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
fc.MustSetBit(1, 4321, 65537, 0)
// Union the handles together.
if result, err := fc.Union(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, result); n != 2 {
t.Fatalf("unexpected union bit count: %d", n)
}
}
// Ensure unioning a bitmap with an empty bitmap returns a single bit count.
func TestFragmentContainer_Union_Empty(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
// Union the handles together.
if result, err := fc.Union(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, result); n != 1 {
t.Fatalf("unexpected empty union bit count: %d", n)
}
}
// Ensure the bits in two bitmaps can be intersected.
func TestFragmentContainer_Intersect(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
fc.MustSetBit(1, 4321, 65537, 0)
// Intersect the handles together.
if result, err := fc.Intersect(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, result); n != 0 {
t.Fatalf("unexpected intersect bit count: %d", n)
}
}
// Ensure the bits in two bitmaps can be diffed.
func TestFragmentContainer_Difference(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
fc.MustSetBit(1, 4321, 65537, 0)
// Compute the difference between the handles.
if result, err := fc.Difference(1, []pilosa.BitmapHandle{fc.MustGet(1, 1234), fc.MustGet(1, 4321)}); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, result); n != 1 {
t.Fatalf("unexpected difference bit count: %s", err)
}
}
// Ensure bitmaps can be marshaled and unmarshaled to bytes.
func TestFragmentContainer_Bytes(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
// Count bits and marshal to bytes.
beforeN := fc.MustCount(1, 1234)
buf, err := fc.GetBytes(1, 1234)
if err != nil {
t.Fatal(err)
}
// Marshal bytes back to a bitmap and re-count.
bh2, err := fc.FromBytes(1, buf)
if err != nil {
t.Fatal(err)
}
afterN := fc.MustCount(1, bh2)
// Ensure the original bit count matches the new bitmap's bit count.
if beforeN != afterN {
t.Fatalf("unexpected bit count: before=%d, after=%d", beforeN, afterN)
}
}
// Ensure an empty bitmap can be returned.
func TestFragmentContainer_Empty(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
bh, err := fc.Empty(1)
if err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, bh); n != 0 {
t.Fatalf("unexpected bit count: %d", n)
}
}
// Ensure a list of bitmap handles can be returned.
func TestFragmentContainer_GetList(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
fc.MustSetBit(1, 1234, 1, 0)
fc.MustSetBit(1, 4321, 65537, 0)
a, err := fc.GetList(1, []uint64{1234, 4321, 789})
if err != nil {
t.Fatal(err)
}
// Compute the union to ensure they're the correct bitmaps.
if res, err := fc.Union(1, a); err != nil {
t.Fatal(err)
} else if n := fc.MustCount(1, res); n != 2 {
t.Fatalf("unexpected bit count: %d", n)
}
}
// Ensure brand bitmaps can perform a small number of set bits.
func TestFragmentContainer_SetBit_Brand_Small(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "b.n", 0, 2)
for i := uint64(0); i < 1000; i++ {
fc.SetBit(2, 1029, i, 0)
}
}
// Ensure the top n can be computed for a brand.
func TestFragmentContainer_TopN_Brand(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "b.n", 0, 2)
// Set bits on the bitmap.
fc.MustSetBit(2, uint64(1), 1, 2)
fc.MustSetBit(2, uint64(1), 2, 2)
fc.MustSetBit(2, uint64(1), 3, 2)
fc.MustSetBit(2, uint64(2), 1, 2)
fc.MustSetBit(2, uint64(2), 2, 2)
fc.MustSetBit(2, uint64(3), 1, 2)
// Retrieve the bitmap handle for bitmap 1.
bh := fc.MustGet(2, uint64(1))
// Compute the top-n.
if results, err := fc.TopN(2, bh, 4, []uint64{2}); err != nil {
t.Fatal(err)
} else if results[0].Key != 1 {
t.Fatalf("unexpected key: %d", results[0].Key)
} else if results[0].Count != 3 {
t.Fatalf("unexpected value: %d", results[0].Count)
}
}
// Ensure a fragment can be cleared.
func TestFragmentContainer_Clear(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "general", 0, 1)
// Compute the top-n.
if res, err := fc.Clear(1); err != nil {
t.Fatal(err)
} else if res != true {
t.Fatalf("unexpected result: %v", res)
}
}
// Ensure a fragment can be marshaled and unmarshaled to and from bytes.
func TestFragmentContainer_FromBytes(t *testing.T) {
fc := NewFragmentContainer()
fc.AddFragment("25", "b.n", 0, 2)
// Set bits for a bitmap.
for i := 0; i < 4096; i++ {
fc.MustSetBit(2, 1, uint64(i), 0)
}
// Marshal to bytes.
buf, err := fc.GetBytes(2, fc.MustGet(2, 1))
if err != nil {
t.Fatal(err)
}
// Load a bitmap from compressed data.
bh, err := fc.FromBytes(2, buf)
if err != nil {
t.Fatal(err)
}
// Load and count bits.
if n := fc.MustCount(2, bh); n != 4096 {
t.Fatalf("unexpected bit count: %d", n)
}
}
// FragementContainer is a test wrapper for pilosa.FragmentContainer.
type FragmentContainer struct {
*pilosa.FragmentContainer
}
// NewFragmentContainer returns a new instance of FragmentContainer.
func NewFragmentContainer() *FragmentContainer {
return &FragmentContainer{pilosa.NewFragmentContainer()}
}
// MustGet retrieves a bitmap by id. Panic on error.
func (fc *FragmentContainer) MustGet(frag_id pilosa.SUUID, bitmap_id uint64) pilosa.BitmapHandle {
bh, err := fc.Get(frag_id, bitmap_id)
if err != nil {
panic(err)
}
return bh
}
// MustSetBit sets a bit in a bitmap. Panic on error.
func (fc *FragmentContainer) MustSetBit(frag_id pilosa.SUUID, bitmap_id uint64, pos uint64, category uint64) bool {
changed, err := fc.SetBit(frag_id, bitmap_id, pos, category)
if err != nil {
panic(err)
}
return changed
}
// MustClear clears a fragment. Panic on error.
func (fc *FragmentContainer) MustClear(fragmentID pilosa.SUUID) bool {
v, err := fc.Clear(fragmentID)
if err != nil {
panic(err)
}
return v
}
// MustCount returns the number of set bits in a bitmap. Panic on error.
func (fc *FragmentContainer) MustCount(frag_id pilosa.SUUID, bitmap pilosa.BitmapHandle) uint64 {
v, err := fc.Count(frag_id, bitmap)
if err != nil {
panic(err)
}
return v
}

8
fragment_test.go Normal file
View file

@ -0,0 +1,8 @@
package pilosa_test
import (
"github.com/umbel/pilosa"
)
// SliceWidth is a helper reference to use when testing.
const SliceWidth = pilosa.SliceWidth

174
general.go
View file

@ -1,174 +0,0 @@
package pilosa
import (
"encoding/json"
"fmt"
log "github.com/cihub/seelog"
"github.com/golang/groupcache/lru"
)
type General struct {
bitmap_cache *lru.Cache
keys map[uint64]interface{}
db string
frame string
slice int
storage Storage
}
func NewGeneral(db string, frame string, slice int, s Storage) *General {
f := new(General)
f.storage = s
f.frame = frame
f.slice = slice
f.db = db
f.Clear()
f.keys = make(map[uint64]interface{})
return f
}
func (self *General) Clear() bool {
self.bitmap_cache = lru.New(50000)
self.bitmap_cache.OnEvicted = self.OnEvicted
return true
}
func (self *General) Exists(bitmap_id uint64) bool {
_, ok := self.bitmap_cache.Get(bitmap_id)
return ok
}
func (self *General) Get(bitmap_id uint64) *Bitmap {
bm, ok := self.bitmap_cache.Get(bitmap_id)
if ok && bm != nil {
return bm.(*Bitmap)
}
bm, _ = self.storage.Fetch(bitmap_id, self.db, self.frame, self.slice)
self.bitmap_cache.Add(bitmap_id, bm)
self.keys[bitmap_id] = 0
return bm.(*Bitmap)
}
func (self *General) SetBit(bitmap_id uint64, bit_pos uint64, filter uint64) bool {
bm := self.Get(bitmap_id)
change, chunk, address := bm.SetBit(bit_pos)
if change {
val := chunk.Value[address.BlockIndex]
self.storage.StoreBit(bitmap_id, self.db, self.frame, self.slice, filter, address.ChunkKey, int32(address.BlockIndex), val, bm.Count())
}
return change
}
func (self *General) TopN(b *Bitmap, n int, categories []uint64) []Pair {
var empty []Pair
return empty
}
func (self *General) Store(bitmap_id uint64, bm *Bitmap, filter uint64) error {
if err := self.storage.Store(bitmap_id, self.db, self.frame, self.slice, filter, bm); err != nil {
return err
}
self.bitmap_cache.Add(bitmap_id, bm)
self.keys[bitmap_id] = 0
return nil
}
func (self *General) OnEvicted(key lru.Key, value interface{}) {
delete(self.keys, key.(uint64))
}
func (self *General) Stats() interface{} {
stats := map[string]interface{}{
"total size of cache in items": self.bitmap_cache.Len()}
return stats
}
func (self *General) getFileName() string {
base := FragmentBase
if base == "" {
base = "."
}
return fmt.Sprintf("%s/%s.%s.%d", base, self.db, self.frame, self.slice)
}
func (self *General) Persist() error {
log.Warn("General Persist")
w, err := createFile(self.getFileName())
if err != nil {
log.Warn("Error saving:", err)
return err
}
defer w.Close()
defer self.storage.Close()
results := make([]uint64, len(self.keys))
i := 0
for k, _ := range self.keys { // map[uint64]*Rank
results[i] = k
i += 1
}
encoder := json.NewEncoder(w)
return encoder.Encode(results)
}
func (self *General) Load(f *Fragment) {
log.Warn("General Load")
r, err := openFile(self.getFileName())
if err != nil {
log.Warn("NO General Init File:", self.getFileName())
return
}
dec := json.NewDecoder(r)
var keys []uint64
if err := dec.Decode(&keys); err != nil {
return
}
for _, k := range keys {
self.Get(k)
}
}
func (self *General) TopNAll(n int, categories []uint64) []Pair {
results := make([]Pair, 0, 0)
count := 0
for k, _ := range self.keys {
if count >= n {
break
}
bm := self.Get(k)
results = append(results, Pair{k, bm.Count()})
count++
}
return results
}
func (self *General) ClearBit(bitmap_id uint64, bit_pos uint64) bool {
bm := self.Get_nocache(bitmap_id)
if bm.Count() == 0 {
return false
}
change, chunk, address := bm.ClearBit(bit_pos)
if change {
val := chunk.Value[address.BlockIndex]
if val == 0 {
self.storage.RemoveBit(bitmap_id, self.db, self.frame, self.slice, uint64(0), address.ChunkKey, int32(address.BlockIndex), bm.Count())
} else {
self.storage.StoreBit(bitmap_id, self.db, self.frame, self.slice, uint64(0), address.ChunkKey, int32(address.BlockIndex), val, bm.Count())
}
}
return change
}
func (self *General) Get_nocache(bitmap_id uint64) *Bitmap {
bm, ok := self.bitmap_cache.Get(bitmap_id)
if ok && bm != nil {
return bm.(*Bitmap)
}
bm, _ = self.storage.Fetch(bitmap_id, self.db, self.frame, self.slice)
return bm.(*Bitmap)
}

261
handler.go Normal file
View file

@ -0,0 +1,261 @@
package pilosa
import (
"encoding/json"
"errors"
"expvar"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
"strings"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa/internal"
"github.com/umbel/pilosa/pql"
)
// Handler represents an HTTP handler.
type Handler struct {
// The execution engine for running queries.
Executor interface {
Execute(db string, query *pql.Query, slices []uint64) (interface{}, error)
}
// The version to report on the /version endpoint.
Version string
// The writer for any logging.
LogOutput io.Writer
}
// NewHandler returns a new instance of Handler with a default logger.
func NewHandler() *Handler {
return &Handler{
Version: Version,
LogOutput: os.Stderr,
}
}
// ServeHTTP handles an HTTP request.
func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/query":
h.handleQuery(w, r)
case "/version":
h.handleVersion(w, r)
case "/debug/vars":
h.handleExpvar(w, r)
default:
http.NotFound(w, r)
}
}
// handleQuery handles /query requests.
func (h *Handler) handleQuery(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case "POST":
h.handlePostQuery(w, r)
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
}
}
// handlePostQuery handles /query requests.
func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
// Parse incoming request.
db, query, slices, err := h.readQueryRequest(r)
if err != nil {
w.WriteHeader(http.StatusBadRequest)
h.writeQueryResponse(w, r, nil, err)
return
}
// Parse query string.
q, err := pql.NewParser(strings.NewReader(query)).Parse()
if err != nil {
w.WriteHeader(http.StatusBadRequest)
h.writeQueryResponse(w, r, nil, err)
return
}
// Execute the query.
res, e := h.Executor.Execute(db, q, slices)
// Set appropriate status code, if there is an error.
if e != nil {
w.WriteHeader(http.StatusInternalServerError)
}
// Write response back to client.
if err := h.writeQueryResponse(w, r, res, e); err != nil {
h.logger().Printf("write query response error: %s", err)
}
}
// readQueryRequest parses an query parameters from r.
func (h *Handler) readQueryRequest(r *http.Request) (db, query string, slices []uint64, err error) {
switch r.Header.Get("Content-Type") {
case "application/x-protobuf":
return h.readProtobufQueryRequest(r)
default:
return h.readURLQueryRequest(r)
}
}
// readProtobufQueryRequest parses query parameters in protobuf from r.
func (h *Handler) readProtobufQueryRequest(r *http.Request) (db, query string, slices []uint64, err error) {
// Slurp the body.
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
// Unmarshal into object.
var req internal.QueryRequest
if err = proto.Unmarshal(body, &req); err != nil {
return
}
return req.GetDB(), req.GetQuery(), req.GetSlices(), nil
}
// readURLQueryRequest parses query parameters from URL parameters from r.
func (h *Handler) readURLQueryRequest(r *http.Request) (db, query string, slices []uint64, err error) {
q := r.URL.Query()
// Read DB argument.
db = q.Get("db")
// Parse query string.
buf, err := ioutil.ReadAll(r.Body)
if err != nil {
return
}
query = string(buf)
// Parse list of slices.
slices, err = parseUint64Slice(q.Get("slices"))
if err != nil {
err = errors.New("invalid slice argument")
return
}
return
}
// writeQueryResponse writes the response from the executor to w.
func (h *Handler) writeQueryResponse(w http.ResponseWriter, r *http.Request, res interface{}, err error) error {
if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") {
return h.writeProtobufQueryResponse(w, res, err)
}
return h.writeJSONQueryResponse(w, res, err)
}
// writeProtobufQueryResponse writes the response from the executor to w as protobuf.
func (h *Handler) writeProtobufQueryResponse(w http.ResponseWriter, res interface{}, e error) error {
var resp internal.QueryResponse
// Set the result on the appropriate field.
if res != nil {
switch res := res.(type) {
case *Bitmap:
resp.Bitmap = encodeBitmap(res)
case Pairs:
resp.Pairs = encodePairs(res)
case uint64:
resp.N = proto.Uint64(res)
default:
panic(fmt.Sprintf("invalid query response type: %T", res))
}
}
// Set the error if there is one.
if e != nil {
resp.Err = proto.String(e.Error())
}
// Encode response.
buf, err := proto.Marshal(&resp)
if err != nil {
return err
}
// Write response back to client.
if _, err := w.Write(buf); err != nil {
return err
}
return nil
}
// writeJSONQueryResponse writes the response from the executor to w as JSON.
func (h *Handler) writeJSONQueryResponse(w http.ResponseWriter, res interface{}, e error) error {
var o struct {
Result interface{} `json:"result,omitempty"`
Error string `json:"error,omitempty"`
}
o.Result = res
if e != nil {
o.Error = e.Error()
}
// Otherwise marshal the result as JSON.
return json.NewEncoder(w).Encode(o)
}
// handleGetVersion handles /version requests.
func (h *Handler) handleVersion(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(struct {
Version string `json:"version"`
}{
Version: h.Version,
}); err != nil {
h.logger().Printf("write version response error: %s", err)
}
}
// handleExpvar handles /debug/vars requests.
func (h *Handler) handleExpvar(w http.ResponseWriter, r *http.Request) {
// Copied from $GOROOT/src/expvar/expvar.go
w.Header().Set("Content-Type", "application/json; charset=utf-8")
fmt.Fprintf(w, "{\n")
first := true
expvar.Do(func(kv expvar.KeyValue) {
if !first {
fmt.Fprintf(w, ",\n")
}
first = false
fmt.Fprintf(w, "%q: %s", kv.Key, kv.Value)
})
fmt.Fprintf(w, "\n}\n")
}
// logger returns a logger for the handler.
func (h *Handler) logger() *log.Logger {
return log.New(h.LogOutput, "", log.LstdFlags)
}
// parseUint64Slice returns a slice of uint64s from a comma-delimited string.
func parseUint64Slice(s string) ([]uint64, error) {
var a []uint64
for _, str := range strings.Split(s, ",") {
// Ignore blanks.
if str == "" {
continue
}
// Parse number.
num, err := strconv.ParseUint(str, 10, 64)
if err != nil {
return nil, err
}
a = append(a, num)
}
return a, nil
}

373
handler_test.go Normal file
View file

@ -0,0 +1,373 @@
package pilosa_test
import (
"bytes"
"errors"
"io"
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"strings"
"testing"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/internal"
"github.com/umbel/pilosa/pql"
)
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_NotFound(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/no_such_path", nil))
if w.Code != http.StatusNotFound {
t.Fatalf("invalid status: %d", w.Code)
}
}
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `count(get(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return uint64(100), nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("count( get( 100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":100}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can accept arguments via protobufs.
func TestHandler_Query_Args_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `count(get(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return uint64(100), nil
}
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
DB: proto.String("db0"),
Query: proto.String("count(get(100))"),
Slices: []uint64{0, 1},
})
if err != nil {
t.Fatal(err)
}
// Generate protobuf request.
req := MustNewHTTPRequest("POST", "/query", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/x-protobuf")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Ensure the handler returns an error when parsing bad arguments.
func TestHandler_Query_Args_Err(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=a,b", strings.NewReader("get(100)")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query with a uint64 response as JSON.
func TestHandler_Query_Uint64_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return uint64(100), nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("count( get( 100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":100}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query with a uint64 response as protobufs.
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return uint64(100), nil
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader("count(get(100))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if resp.GetN() != 100 {
t.Fatalf("unexpected n: %d", resp.GetN())
}
}
// Ensure the handler can execute a query that returns a bitmap as JSON.
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
bm := pilosa.NewBitmap()
bm.SetBit(1)
bm.SetBit(3)
bm.SetBit(66)
bm.SetBit(pilosa.SliceWidth + 1)
return bm, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader("get(100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":{"chunks":[{"Key":0,"Value":[10,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]},{"Key":32,"Value":[2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]}]}}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query that returns a bitmap as protobuf.
func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
bm := pilosa.NewBitmap()
bm.SetBit(1)
bm.SetBit(pilosa.SliceWidth + 1)
return bm, nil
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader("get(100)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.GetBitmap().GetChunks(); len(a) != 2 {
t.Fatalf("unexpected bitmap chunk length: %d", len(a))
}
}
// Ensure the handler can execute a query that returns pairs as JSON.
func TestHandler_Query_Pairs_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return pilosa.Pairs{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`top-n(frame=x, n=2)`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"result":[{"key":1,"count":2},{"key":3,"count":4}]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query that returns pairs as protobuf.
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return pilosa.Pairs{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
}, nil
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader(`top-n(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if a := resp.GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
}
// Ensure the handler can return an error as JSON.
func TestHandler_Query_Err_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query", strings.NewReader(`get(100)`)))
if w.Code != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can return an error as protobuf.
func TestHandler_Query_Err_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/query", strings.NewReader(`top-n(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if s := resp.GetErr(); s != `marker` {
t.Fatalf("unexpected error: %s", s)
}
}
// Ensure the handler returns "method not allowed" for non-POST queries.
func TestHandler_Query_MethodNotAllowed(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("GET", "/query", nil))
if w.Code != http.StatusMethodNotAllowed {
t.Fatalf("invalid status: %d", w.Code)
}
}
// Ensure the handler returns an error if there is a parsing error..
func TestHandler_Query_ErrParse(t *testing.T) {
h := NewHandler()
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/query?db=db0&slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"function not found: bad_fn occurred at line 1, char 1"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can retrieve the version.
func TestHandler_Version(t *testing.T) {
h := NewHandler()
h.Version = "1.0.0"
w := httptest.NewRecorder()
r := MustNewHTTPRequest("GET", "/version", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"version":"1.0.0"}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// Ensure the handler can return expvars without panicking.
func TestHandler_Expvars(t *testing.T) {
h := NewHandler()
w := httptest.NewRecorder()
r := MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Handler represents a test wrapper for pilosa.Handler.
type Handler struct {
*pilosa.Handler
Executor HandlerExecutor
}
// NewHandler returns a new instance of Handler.
func NewHandler() *Handler {
h := &Handler{
Handler: pilosa.NewHandler(),
}
h.Handler.Executor = &h.Executor
return h
}
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(db string, query *pql.Query, slices []uint64) (interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return c.ExecuteFn(db, query, slices)
}
// Server represents a test wrapper for httptest.Server.
type Server struct {
*httptest.Server
Handler *Handler
}
// NewServer returns a test server running on a random port.
func NewServer() *Server {
s := &Server{
Handler: NewHandler(),
}
s.Server = httptest.NewServer(s.Handler.Handler)
return s
}
// Host returns the hostname of the running server.
func (s *Server) Host() string {
u, err := url.Parse(s.URL)
if err != nil {
panic(err)
}
return u.Host
}
// MustNewHTTPRequest creates a new HTTP request. Panic on error.
func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request {
req, err := http.NewRequest(method, urlStr, body)
if err != nil {
panic(err)
}
return req
}

88
hold/hold.go
View file

@ -1,88 +0,0 @@
package hold
import (
"errors"
"time"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
)
type holdchan chan interface{}
type gethold struct {
id *pilosa.GUID
reply chan holdchan
}
type delhold struct {
id *pilosa.GUID
}
type Holder struct {
data map[pilosa.GUID]holdchan
getchan chan gethold
delchan chan delhold
}
func NewHolder() *Holder {
return &Holder{
data: make(map[pilosa.GUID]holdchan),
getchan: make(chan gethold),
delchan: make(chan delhold),
}
}
func (self *Holder) DelChan(id *pilosa.GUID) {
log.Trace("Holder.DelChan", id)
req := delhold{id}
self.delchan <- req
}
func (self *Holder) GetChan(id *pilosa.GUID) holdchan {
log.Trace("Holder.GetChan", id)
reply := make(chan holdchan)
req := gethold{id, reply}
self.getchan <- req
return <-reply
}
func (self *Holder) Get(id *pilosa.GUID, timeout time.Duration) (interface{}, error) {
log.Trace("Holder.Get", id, timeout.String())
ch := self.GetChan(id)
select {
case val := <-ch:
return val, nil
case <-time.After(timeout):
self.DelChan(id)
return nil, errors.New("Timeout getting from holder")
}
}
func (self *Holder) Set(id *pilosa.GUID, value interface{}, timeout time.Duration) {
log.Trace("Holder.Set", id, value, timeout.String())
ch := self.GetChan(id)
go func() {
select {
case ch <- value:
case <-time.After(timeout):
}
self.DelChan(id)
}()
}
func (self *Holder) Run() {
var greq gethold
var dreq delhold
for {
select {
case greq = <-self.getchan:
item, ok := self.data[*greq.id]
if !ok {
item = make(holdchan)
self.data[*greq.id] = item
}
greq.reply <- item
case dreq = <-self.delchan:
delete(self.data, *dreq.id)
}
}
}

41
hold/hold_test.go
View file

@ -1,41 +0,0 @@
package hold_test
import (
"testing"
"time"
"github.com/umbel/pilosa/hold"
"github.com/umbel/pilosa/util"
)
// Ensure hold can get a value that has been set.
func TestHold_Get(t *testing.T) {
h := hold.NewHolder()
go h.Run()
id := util.RandomUUID()
h.Set(&id, "derp", 10)
if v, err := h.Get(&id, 10); err != nil {
t.Fatal(err)
} else if v != "derp" {
t.Fatalf("unexpected value: %v", v)
}
}
// Ensure hold can wait for a value that has not been set yet.
func TestHold_Get_Delay(t *testing.T) {
h := hold.NewHolder()
go h.Run()
id := util.RandomUUID()
go func() {
time.Sleep(500 * time.Millisecond)
h.Set(&id, "derpsy", 10)
}()
if v, err := h.Get(&id, 10); err != nil {
t.Fatal(err)
} else if v != "derpsy" {
t.Fatalf("unexpected value: %v", v)
}
}

53
index.go Normal file
View file

@ -0,0 +1,53 @@
package pilosa
import (
"sync"
)
// Index represents a container for fragments.
type Index struct {
mu sync.Mutex
sliceN uint64
fragments map[fragmentKey]*Fragment
}
// NewIndex returns a new instance of Index.
func NewIndex() *Index {
return &Index{
fragments: make(map[fragmentKey]*Fragment),
}
}
// SliceN returs the total number of slices managed by the index.
func (i *Index) SliceN() uint64 {
i.mu.Lock()
defer i.mu.Unlock()
return i.sliceN
}
// Fragment returns the fragment for a database, frame & slice.
// The fragment is created if it doesn't already exist.
func (i *Index) Fragment(db, frame string, slice uint64) *Fragment {
i.mu.Lock()
defer i.mu.Unlock()
// Track the highest slice.
if slice > i.sliceN {
i.sliceN = slice
}
// Create fragment, if not exists.
key := fragmentKey{db, frame, slice}
if i.fragments[key] == nil {
i.fragments[key] = NewFragment(db, frame, slice)
}
return i.fragments[key]
}
// fragmentKey is the map key for fragment look ups.
type fragmentKey struct {
db string
frame string
slice uint64
}

56
internal/internal.go Normal file
View file

@ -0,0 +1,56 @@
package internal
import (
"io"
"io/ioutil"
"github.com/gogo/protobuf/proto"
)
type Request proto.Message
type Response proto.Message
// Encoder encodes messages to a writer.
type Encoder struct {
w io.Writer
}
// NewEncoder returns a new instance of Encoder.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{w: w}
}
// Encode marshals m into bytes and writes them to r.
func (enc *Encoder) Encode(pb proto.Message) error {
buf, err := proto.Marshal(pb)
if err != nil {
return err
}
if _, err := enc.w.Write(buf); err != nil {
return err
}
return nil
}
// Decoder decodes messages from a reader.
type Decoder struct {
r io.Reader
}
// NewDecoder returns a new instance of Decoder.
func NewDecoder(r io.Reader) *Decoder {
return &Decoder{r: r}
}
// Decode reads all bytes from the reader and unmarshals them into pb.
func (dec *Decoder) Decode(pb proto.Message) error {
buf, err := ioutil.ReadAll(dec.r)
if err != nil {
return err
}
return proto.Unmarshal(buf, pb)
}

164
internal/internal.pb.go Normal file
View file

@ -0,0 +1,164 @@
// Code generated by protoc-gen-gogo.
// source: internal/internal.proto
// DO NOT EDIT!
/*
Package internal is a generated protocol buffer package.
It is generated from these files:
internal/internal.proto
It has these top-level messages:
Bitmap
Chunk
Pair
QueryRequest
QueryResponse
*/
package internal
import proto "github.com/gogo/protobuf/proto"
import math "math"
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = math.Inf
type Bitmap struct {
Chunks []*Chunk `protobuf:"bytes,1,rep" json:"Chunks,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Bitmap) Reset() { *m = Bitmap{} }
func (m *Bitmap) String() string { return proto.CompactTextString(m) }
func (*Bitmap) ProtoMessage() {}
func (m *Bitmap) GetChunks() []*Chunk {
if m != nil {
return m.Chunks
}
return nil
}
type Chunk struct {
Key *uint64 `protobuf:"varint,1,req" json:"Key,omitempty"`
Value []uint64 `protobuf:"varint,2,rep" json:"Value,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Chunk) Reset() { *m = Chunk{} }
func (m *Chunk) String() string { return proto.CompactTextString(m) }
func (*Chunk) ProtoMessage() {}
func (m *Chunk) GetKey() uint64 {
if m != nil && m.Key != nil {
return *m.Key
}
return 0
}
func (m *Chunk) GetValue() []uint64 {
if m != nil {
return m.Value
}
return nil
}
type Pair struct {
Key *uint64 `protobuf:"varint,1,req" json:"Key,omitempty"`
Count *uint64 `protobuf:"varint,2,req" json:"Count,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *Pair) Reset() { *m = Pair{} }
func (m *Pair) String() string { return proto.CompactTextString(m) }
func (*Pair) ProtoMessage() {}
func (m *Pair) GetKey() uint64 {
if m != nil && m.Key != nil {
return *m.Key
}
return 0
}
func (m *Pair) GetCount() uint64 {
if m != nil && m.Count != nil {
return *m.Count
}
return 0
}
type QueryRequest struct {
DB *string `protobuf:"bytes,1,req" json:"DB,omitempty"`
Query *string `protobuf:"bytes,2,req" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,3,rep" json:"Slices,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
func (m *QueryRequest) String() string { return proto.CompactTextString(m) }
func (*QueryRequest) ProtoMessage() {}
func (m *QueryRequest) GetDB() string {
if m != nil && m.DB != nil {
return *m.DB
}
return ""
}
func (m *QueryRequest) GetQuery() string {
if m != nil && m.Query != nil {
return *m.Query
}
return ""
}
func (m *QueryRequest) GetSlices() []uint64 {
if m != nil {
return m.Slices
}
return nil
}
type QueryResponse struct {
Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"`
Bitmap *Bitmap `protobuf:"bytes,2,opt" json:"Bitmap,omitempty"`
N *uint64 `protobuf:"varint,3,opt" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,4,rep" json:"Pairs,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
func (m *QueryResponse) Reset() { *m = QueryResponse{} }
func (m *QueryResponse) String() string { return proto.CompactTextString(m) }
func (*QueryResponse) ProtoMessage() {}
func (m *QueryResponse) GetErr() string {
if m != nil && m.Err != nil {
return *m.Err
}
return ""
}
func (m *QueryResponse) GetBitmap() *Bitmap {
if m != nil {
return m.Bitmap
}
return nil
}
func (m *QueryResponse) GetN() uint64 {
if m != nil && m.N != nil {
return *m.N
}
return 0
}
func (m *QueryResponse) GetPairs() []*Pair {
if m != nil {
return m.Pairs
}
return nil
}
func init() {
}

28
internal/internal.proto Normal file
View file

@ -0,0 +1,28 @@
package internal;
message Bitmap {
repeated Chunk Chunks = 1;
}
message Chunk {
required uint64 Key = 1;
repeated uint64 Value = 2;
}
message Pair {
required uint64 Key = 1;
required uint64 Count = 2;
}
message QueryRequest {
required string DB = 1;
required string Query = 2;
repeated uint64 Slices = 3;
}
message QueryResponse {
optional string Err = 1;
optional Bitmap Bitmap = 2;
optional uint64 N = 3;
repeated Pair Pairs = 4;
}

122
pilosa.go
View file

@ -1,122 +1,6 @@
package pilosa
import (
"encoding/binary"
"encoding/hex"
"fmt"
"math/rand"
"strings"
"time"
//go:generate protoc --gogo_out=. internal/internal.proto
"github.com/gocql/gocql"
)
var counter = uint64(0)
func init() {
rand.Seed(time.Now().UTC().UnixNano())
}
// SUUID represents a sequential UUID.
type SUUID uint64
// SUUID returns a new SUUID.
func NewSUUID() SUUID {
millis := uint64(time.Now().UTC().UnixNano())
id := millis << (64 - 41)
id |= uint64(rand.Intn(128)) << (64 - 41 - 13)
id |= counter % 1024
counter += 1
return SUUID(id)
}
// String returns a string representation of id.
func (id SUUID) String() string {
var buf [8]byte
binary.BigEndian.PutUint64(buf[:], uint64(id))
return hex.EncodeToString(buf[:])
}
// ParseSUUID parses s into an SUUID.
func ParseSUUID(s string) SUUID {
if n := len(s); n < 16 {
s = strings.Repeat("0", 16-n) + s
}
b, _ := hex.DecodeString(s)
return SUUID(binary.BigEndian.Uint64(b))
}
// GUID represents a globally unique identifier.
type GUID [16]byte
// UnmarshalText parses a text value into a GUID.
// This is used by the TOML parser.
func (id *GUID) UnmarshalText(text []byte) error {
v, err := ParseGUID(string(text))
if err != nil {
return err
}
*id = v
return nil
}
func (id GUID) String() string {
var offsets = [...]int{0, 2, 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 28, 30, 32, 34}
const hexString = "0123456789abcdef"
r := make([]byte, 36)
for i, b := range id {
r[offsets[i]] = hexString[b>>4]
r[offsets[i]+1] = hexString[b&0xF]
}
r[8] = '-'
r[13] = '-'
r[18] = '-'
r[23] = '-'
return string(r)
}
// Equals returns true if id equals other.
func (id *GUID) Equals(other *GUID) bool {
for i, v := range id {
if v != other[i] {
return false
}
}
return true
}
// NewGUID returns a random GUID.
func NewGUID() GUID {
uid, _ := gocql.RandomUUID()
var id GUID
copy(id[:], uid[:])
return id
}
// ParseGUID parses s into a GUID.
func ParseGUID(s string) (GUID, error) {
var u GUID
j := 0
for _, r := range s {
switch {
case r == '-' && j&1 == 0:
continue
case r >= '0' && r <= '9' && j < 32:
u[j/2] |= byte(r-'0') << uint(4-j&1*4)
case r >= 'a' && r <= 'f' && j < 32:
u[j/2] |= byte(r-'a'+10) << uint(4-j&1*4)
case r >= 'A' && r <= 'F' && j < 32:
u[j/2] |= byte(r-'A'+10) << uint(4-j&1*4)
default:
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
j += 1
}
if j != 32 {
return GUID{}, fmt.Errorf("invalid GUID %q", s)
}
return u, nil
}
// Version represents the current running version of Pilosa.
var Version string

51
pilosa_test.go
View file

@ -1,51 +0,0 @@
package pilosa_test
import (
"fmt"
"testing"
"github.com/umbel/pilosa"
)
// Ensure id can be parsed from string.
func TestSUUID_Small(t *testing.T) {
if v := pilosa.ParseSUUID("1"); v != 1 {
t.Fatalf("unexpected SUUID: %v", v)
}
}
// Ensure generated IDs are unique.
func TestSUUID_Unique(t *testing.T) {
a, b := pilosa.NewSUUID(), pilosa.NewSUUID()
if a == b {
t.Fatalf("ids should be unique: %v != %v", a, b)
}
}
// Ensure ids can be converted to and from hex.
func TestSUUID_Hex(t *testing.T) {
a := pilosa.NewSUUID()
b := pilosa.ParseSUUID(a.String())
if a != b {
t.Fatalf("ids not equal: %v != %v", a, b)
}
}
// Ensure ids can be generated in sequence.
func TestSUUID_Multiple(t *testing.T) {
for i := 0; i < 10; i++ {
println(pilosa.NewSUUID().String())
}
}
// Ensure a random GUID can be converted to a string.
func TestGUID_String(t *testing.T) {
fmt.Println(pilosa.NewGUID().String())
}
func BenchmarkSUUID(b *testing.B) {
// run the Fib function b.N times
for n := 0; n < b.N; n++ {
pilosa.NewSUUID()
}
}

24
popcnt_generic.go
View file

@ -2,22 +2,8 @@
package pilosa
func popcntSlice(s []uint64) uint64 {
return popcntSliceGo(s)
}
func popcntMaskSlice(s, m []uint64) uint64 {
return popcntMaskSliceGo(s, m)
}
func popcntAndSlice(s, m []uint64) uint64 {
return popcntAndSliceGo(s, m)
}
func popcntOrSlice(s, m []uint64) uint64 {
return popcntOrSliceGo(s, m)
}
func popcntXorSlice(s, m []uint64) uint64 {
return popcntXorSliceGo(s, m)
}
func popcntSlice(s []uint64) uint64 { return popcntSliceGo(s) }
func popcntMaskSlice(s, m []uint64) uint64 { return popcntMaskSliceGo(s, m) }
func popcntAndSlice(s, m []uint64) uint64 { return popcntAndSliceGo(s, m) }
func popcntOrSlice(s, m []uint64) uint64 { return popcntOrSliceGo(s, m) }
func popcntXorSlice(s, m []uint64) uint64 { return popcntXorSliceGo(s, m) }

228
pql/ast.go Normal file
View file

@ -0,0 +1,228 @@
package pql
import (
"fmt"
"strings"
"time"
)
// Query represents a PQL query.
type Query struct {
Root Call
}
// String returns a string representation of the query.
func (q *Query) String() string { return q.Root.String() }
// Node represents any node in the AST.
type Node interface {
node()
String() string
}
func (*Clear) node() {}
func (*Count) node() {}
func (*Difference) node() {}
func (*Get) node() {}
func (*Intersect) node() {}
func (*Range) node() {}
func (*Set) node() {}
func (*TopN) node() {}
func (*Union) node() {}
// Call represents a function call in the AST.
type Call interface {
Node
call()
}
func (*Clear) call() {}
func (*Count) call() {}
func (*Difference) call() {}
func (*Get) call() {}
func (*Intersect) call() {}
func (*Range) call() {}
func (*Set) call() {}
func (*TopN) call() {}
func (*Union) call() {}
// Calls represents a list of calls.
type Calls []Call
// String returns a string representation of the calls as a comma-delimited list.
func (a Calls) String() string {
args := make([]string, len(a))
for i, c := range a {
args[i] = c.String()
}
return strings.Join(args, ", ")
}
// BitmapCall represents a function call that returns a bitmap.
type BitmapCall interface {
Call
bitmapCall()
}
// BitmapCalls represents a list of bitmap calls.
type BitmapCalls []BitmapCall
// String returns a string representation of the calls as a comma-delimited list.
func (a BitmapCalls) String() string {
args := make([]string, len(a))
for i, c := range a {
args[i] = c.String()
}
return strings.Join(args, ", ")
}
func (*Difference) bitmapCall() {}
func (*Get) bitmapCall() {}
func (*Intersect) bitmapCall() {}
func (*Range) bitmapCall() {}
func (*Union) bitmapCall() {}
// Clear represents a clear() function call.
type Clear struct {
ID uint64
Frame string
Filter uint64
ProfileID uint64
}
// String returns the string representation of the call.
func (c *Clear) String() string {
args := make([]string, 0, 4)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.Filter != 0 {
args = append(args, fmt.Sprintf("filter=%d", c.Filter))
}
if c.ProfileID != 0 {
args = append(args, fmt.Sprintf("profile_id=%d", c.ProfileID))
}
return fmt.Sprintf("clear(%s)", strings.Join(args, ", "))
}
// Count represents a count() function call.
type Count struct {
Input BitmapCall
}
// String returns the string representation of the call.
func (c *Count) String() string {
return fmt.Sprintf("count(%s)", c.Input.String())
}
// Difference represents an difference() function call.
type Difference struct {
Inputs BitmapCalls
}
// String returns the string representation of the call.
func (c *Difference) String() string {
return fmt.Sprintf("difference(%s)", c.Inputs.String())
}
// Get represents a get() function call.
type Get struct {
ID uint64
Frame string
}
// String returns the string representation of the call.
func (c *Get) String() string {
args := make([]string, 0, 2)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
return fmt.Sprintf("get(%s)", strings.Join(args, ", "))
}
// Intersect represents an intersect() function call.
type Intersect struct {
Inputs BitmapCalls
}
// String returns the string representation of the call.
func (c *Intersect) String() string {
return fmt.Sprintf("intersect(%s)", c.Inputs.String())
}
// Range represents a range() function call.
type Range struct {
ID uint64
Frame string
StartTime time.Time
EndTime time.Time
}
// String returns the string representation of the call.
func (c *Range) String() string {
args := make([]string, 0, 2)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if !c.StartTime.IsZero() {
args = append(args, fmt.Sprintf("start=%s", c.StartTime.Format(TimeFormat)))
}
if !c.EndTime.IsZero() {
args = append(args, fmt.Sprintf("end=%s", c.EndTime.Format(TimeFormat)))
}
return fmt.Sprintf("range(%s)", strings.Join(args, ", "))
}
// Set represents a set() function call.
type Set struct {
ID uint64
Frame string
Filter uint64
ProfileID uint64
}
// String returns the string representation of the call.
func (c *Set) String() string {
args := make([]string, 0, 2)
if c.ID != 0 {
args = append(args, fmt.Sprintf("id=%d", c.ID))
}
if c.Frame != "" {
args = append(args, fmt.Sprintf("frame=%s", c.Frame))
}
if c.Filter != 0 {
args = append(args, fmt.Sprintf("filter=%d", c.Filter))
}
if c.ProfileID != 0 {
args = append(args, fmt.Sprintf("profile_id=%d", c.ProfileID))
}
return fmt.Sprintf("set(%s)", strings.Join(args, ", "))
}
// TopN represents a top-n() function call.
type TopN struct {
Frame string
N int
}
// String returns the string representation of the call.
func (c *TopN) String() string { panic("FIXME") }
// Union represents a union() function call.
type Union struct {
Inputs BitmapCalls
}
// String returns the string representation of the call.
func (c *Union) String() string {
return fmt.Sprintf("union(%s)", c.Inputs.String())
}

89
pql/ast_test.go Normal file
View file

@ -0,0 +1,89 @@
package pql_test
import (
"testing"
"time"
"github.com/umbel/pilosa/pql"
)
// Ensure the Clear call can be converted into a string.
func TestClear_String(t *testing.T) {
s := (&pql.Clear{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String()
if s != `clear(id=1, frame=x.n, filter=2, profile_id=3)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Count call can be converted into a string.
func TestCount_String(t *testing.T) {
s := (&pql.Count{Input: &pql.Get{ID: 1, Frame: "x.n"}}).String()
if s != `count(get(id=1, frame=x.n))` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Difference call can be converted into a string.
func TestDifference_String(t *testing.T) {
s := (&pql.Difference{Inputs: pql.BitmapCalls{
&pql.Get{ID: 1, Frame: "x.n"},
&pql.Get{ID: 2},
},
}).String()
if s != `difference(get(id=1, frame=x.n), get(id=2))` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Get call can be converted into a string.
func TestGet_String(t *testing.T) {
s := (&pql.Get{ID: 1, Frame: "x.n"}).String()
if s != `get(id=1, frame=x.n)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Intersect call can be converted into a string.
func TestIntersect_String(t *testing.T) {
s := (&pql.Intersect{Inputs: pql.BitmapCalls{
&pql.Get{ID: 1, Frame: "x.n"},
&pql.Get{ID: 2},
},
}).String()
if s != `intersect(get(id=1, frame=x.n), get(id=2))` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Range call can be converted into a string.
func TestRange_String(t *testing.T) {
s := (&pql.Range{
ID: 1,
Frame: "x.n",
StartTime: time.Unix(0, 0).UTC(),
EndTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
}).String()
if s != `range(id=1, frame=x.n, start=1970-01-01T00:00, end=2000-01-02T03:04)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Set call can be converted into a string.
func TestSet_String(t *testing.T) {
s := (&pql.Set{ID: 1, Frame: "x.n", Filter: 2, ProfileID: 3}).String()
if s != `set(id=1, frame=x.n, filter=2, profile_id=3)` {
t.Fatalf("unexpected string: %s", s)
}
}
// Ensure the Union call can be converted into a string.
func TestUnion_String(t *testing.T) {
s := (&pql.Union{Inputs: pql.BitmapCalls{
&pql.Get{ID: 1, Frame: "x.n"},
&pql.Get{ID: 2},
},
}).String()
if s != `union(get(id=1, frame=x.n), get(id=2))` {
t.Fatalf("unexpected string: %s", s)
}
}

569
pql/parser.go Normal file
View file

@ -0,0 +1,569 @@
package pql
import (
"fmt"
"io"
"strconv"
"strings"
"time"
)
// TimeFormat is the go-style time format used to parse string dates.
const TimeFormat = "2006-01-02T15:04"
// Parser represents a parser for the PQL language.
type Parser struct {
scanner *bufScanner
}
// NewParser returns a new instance of Parser.
func NewParser(r io.Reader) *Parser {
return &Parser{
scanner: newBufScanner(r),
}
}
// ParseString parses s into a query.
func ParseString(s string) (*Query, error) {
return NewParser(strings.NewReader(s)).Parse()
}
// Parse parses the next node in the query.
func (p *Parser) Parse() (*Query, error) {
fn, err := p.parseCall()
if err != nil {
return nil, err
}
return &Query{Root: fn}, nil
}
// parseCall parses the next function call.
func (p *Parser) parseCall() (Call, error) {
tok, pos, lit := p.scanIgnoreWhitespace()
if tok != IDENT {
return nil, &ParseError{Message: fmt.Sprintf("expected identifier, found: %s", lit), Pos: pos}
}
switch lit {
case "count":
return p.parseCountCall()
case "clear":
return p.parseClearCall()
case "difference":
return p.parseDifferenceCall()
case "get":
return p.parseGetCall()
case "intersect":
return p.parseIntersectCall()
case "range":
return p.parseRangeCall()
case "set":
return p.parseSetCall()
case "top-n":
return p.parseTopNCall()
case "union":
return p.parseUnionCall()
default:
return nil, &ParseError{Message: fmt.Sprintf("function not found: %s", lit), Pos: pos}
}
}
// parseClearCall parses a clear() function call.
func (p *Parser) parseClearCall() (*Clear, error) {
c := &Clear{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "filter":
if err := decodeUint64(arg.value, &c.Filter); err != nil {
return nil, parseErrorf(pos, "filter: %s", err)
}
case 3, "profile_id":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profile_id: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseCount parses a count() function call.
func (p *Parser) parseCountCall() (*Count, error) {
c := &Count{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
} else if len(args) != 1 {
return nil, parseErrorf(pos, "count requires one argument")
}
// Copy argument to AST.
input, ok := args[0].value.(BitmapCall)
if !ok {
return nil, parseErrorf(pos, "invalid count arg: %s", args[0].value)
}
c.Input = input
return c, nil
}
// parseDifference parses a difference() function call.
func (p *Parser) parseDifferenceCall() (*Difference, error) {
c := &Difference{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid arg: %v", arg.value)
}
}
return c, nil
}
// parseGetCall parses a get() function call.
func (p *Parser) parseGetCall() (*Get, error) {
c := &Get{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseIntersect parses a intersect() function call.
func (p *Parser) parseIntersectCall() (*Intersect, error) {
c := &Intersect{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid arg: %v", arg.value)
}
}
return c, nil
}
// parseRangeCall parses a range() function call.
func (p *Parser) parseRangeCall() (*Range, error) {
c := &Range{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "start: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "start":
if err := decodeDate(arg.value, &c.StartTime); err != nil {
return nil, parseErrorf(pos, "start: %s", err)
}
case 3, "end":
if err := decodeDate(arg.value, &c.EndTime); err != nil {
return nil, parseErrorf(pos, "end: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseSetCall parses a set() function call.
func (p *Parser) parseSetCall() (*Set, error) {
c := &Set{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "id":
if err := decodeUint64(arg.value, &c.ID); err != nil {
return nil, parseErrorf(pos, "id: %s", err)
}
case 1, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 2, "filter":
if err := decodeUint64(arg.value, &c.Filter); err != nil {
return nil, parseErrorf(pos, "filter: %s", err)
}
case 3, "profile_id":
if err := decodeUint64(arg.value, &c.ProfileID); err != nil {
return nil, parseErrorf(pos, "profile_id: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseTopNCall parses a top-n() function call.
func (p *Parser) parseTopNCall() (*TopN, error) {
c := &TopN{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
switch arg.key {
case 0, "frame":
if err := decodeString(arg.value, &c.Frame); err != nil {
return nil, parseErrorf(pos, "frame: %s", err)
}
case 1, "n":
if err := decodeInt(arg.value, &c.N); err != nil {
return nil, parseErrorf(pos, "n: %s", err)
}
default:
return nil, parseErrorf(pos, "invalid arg: %v", arg.key)
}
}
return c, nil
}
// parseUnion parses a union() function call.
func (p *Parser) parseUnionCall() (*Union, error) {
c := &Union{}
pos := p.pos()
// Scan opening parenthesis.
if err := p.expect(LPAREN); err != nil {
return nil, err
}
// Parse arguments.
args, err := p.parseArgs()
if err != nil {
return nil, err
}
// Copy arguments to AST.
for _, arg := range args {
if v, ok := arg.value.(BitmapCall); ok {
c.Inputs = append(c.Inputs, v)
} else {
return nil, parseErrorf(pos, "invalid arg: %v", arg.value)
}
}
return c, nil
}
// parseArgs arguments to a function call.
func (p *Parser) parseArgs() ([]arg, error) {
var i int
var args []arg
for {
// Parse next argument.
arg, err := p.parseArg()
if err != nil {
return nil, err
}
// If it's a primitive type without a key then index it.
if arg.key == nil {
switch arg.value.(type) {
case uint64, string:
arg.key = i
i++
}
}
// Append argument to list.
args = append(args, arg)
// If next token is a closing parenthesis, then exit.
// Otherwise expect a comma.
if tok, pos, lit := p.scanIgnoreWhitespace(); tok == RPAREN {
break
} else if tok != COMMA {
return nil, parseErrorf(pos, "expected COMMA, found %q", lit)
}
}
return args, nil
}
// parseArg parses a single argument to a function call.
func (p *Parser) parseArg() (arg, error) {
var key, value interface{}
// Read identifier and check if there's a following "=" or "(".
tok, pos, lit := p.scanIgnoreWhitespace()
switch tok {
case IDENT:
// If a left paren immediately follows then it's a function call.
if tok, _, _ := p.scan(); tok == LPAREN {
p.unscan(2)
c, err := p.parseCall()
if err != nil {
return arg{}, err
}
return arg{value: c}, nil
}
// If it's not a left paren, rescan ignoring whitespace and look for "=",
p.unscan(1)
if tok, _, _ := p.scanIgnoreWhitespace(); tok == EQ {
key = lit // keyed arg
} else {
p.unscan(1)
}
default:
p.unscan(1)
}
// Read value token.
tok, pos, lit = p.scanIgnoreWhitespace()
switch tok {
case IDENT, STRING:
value = lit
case NUMBER:
v, err := strconv.ParseUint(lit, 10, 64)
if err != nil {
return arg{}, err
}
value = v
case LBRACK:
panic("FIXME: parse list of integers")
default:
return arg{}, parseErrorf(pos, "invalid value: %q", lit)
}
return arg{key: key, value: value}, nil
}
// scan returns the next token from the scanner.
func (p *Parser) scan() (tok Token, pos Pos, lit string) { return p.scanner.Scan() }
// scanIgnoreWhitespace returns the next non-whitespace token from the scanner.
func (p *Parser) scanIgnoreWhitespace() (tok Token, pos Pos, lit string) {
tok, pos, lit = p.scan()
if tok == WS {
tok, pos, lit = p.scan()
}
return
}
// unscan returns the last n tokens back to the scanner.
func (p *Parser) unscan(n int) {
for i := 0; i < n; i++ {
p.scanner.unscan()
}
}
// expect returns an error if the next token is not exp.
func (p *Parser) expect(exp Token) error {
if tok, pos, lit := p.scan(); tok != exp {
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
}
return nil
}
// expectIgnoreWhitespace returns an error if the next non-whitespace token is not exp.
func (p *Parser) expectIgnoreWhitespace(exp Token) error {
if tok, pos, lit := p.scanIgnoreWhitespace(); tok != exp {
return parseErrorf(pos, "expected %s, found %q", exp.String(), lit)
}
return nil
}
// pos returns the current position.
func (p *Parser) pos() Pos { return p.scanner.pos() }
// arg represents an call argument.
// The key can be the index or the string key.
// The value can be a uint64, []uint64, string, Call, or Calls.
type arg struct {
key interface{}
value interface{}
}
// ParseError represents an error that occurred while parsing a PQL query.
type ParseError struct {
Message string
Pos Pos
}
// Error returns a string representation of e.
func (e *ParseError) Error() string {
return fmt.Sprintf("%s occurred at line %d, char %d", e.Message, e.Pos.Line+1, e.Pos.Char+1)
}
// parseErrorf returns a formatted parse error.
func parseErrorf(pos Pos, format string, args ...interface{}) *ParseError {
return &ParseError{
Message: fmt.Sprintf(format, args...),
Pos: pos,
}
}
// decodeInt type converts v to target.
func decodeInt(v interface{}, target *int) error {
if v, ok := v.(uint64); ok {
*target = int(v)
return nil
}
return fmt.Errorf("invalid int value: %v", v)
}
// decodeUint64 type converts v to target.
func decodeUint64(v interface{}, target *uint64) error {
if v, ok := v.(uint64); ok {
*target = v
return nil
}
return fmt.Errorf("invalid int value: %v", v)
}
// decodeString type converts v to target.
func decodeString(v interface{}, target *string) error {
if v, ok := v.(string); ok {
*target = v
return nil
}
return fmt.Errorf("invalid string value: %v", v)
}
// decodeDate type converts v to target.
func decodeDate(v interface{}, target *time.Time) error {
if v, ok := v.(string); ok {
t, err := time.Parse(TimeFormat, v)
if err != nil {
return fmt.Errorf("invalid date format: %s", v)
}
*target = t
return nil
}
return fmt.Errorf("invalid date value: %v", v)
}

239
pql/parser_test.go Normal file
View file

@ -0,0 +1,239 @@
package pql_test
import (
"reflect"
"testing"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/pql"
)
// Ensure the parser can parse a "clear()" function with keyed args.
func TestParser_Parse_Clear_Key(t *testing.T) {
q, err := pql.ParseString(`clear(id=1, frame="b.n", filter=2, profile_id = 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Clear{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "clear()" function with array args.
func TestParser_Parse_Clear_Array(t *testing.T) {
q, err := pql.ParseString(`clear(1, "b.n", 2, 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Clear{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "count()" function.
func TestParser_Parse_Count(t *testing.T) {
q, err := pql.ParseString(`count(get(1))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Count{
Input: &pql.Get{
ID: 1,
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "difference()" function.
func TestParser_Parse_Difference(t *testing.T) {
q, err := pql.ParseString(`difference(get(1), get(2))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Difference{
Inputs: pql.BitmapCalls{
&pql.Get{ID: 1},
&pql.Get{ID: 2},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "get()" function with keyed args.
func TestParser_Parse_Get_Key(t *testing.T) {
q, err := pql.ParseString(`get(id=1, frame="b.n")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Get{
ID: 1,
Frame: "b.n",
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "get()" function with array args.
func TestParser_Parse_Get_Array(t *testing.T) {
q, err := pql.ParseString(`get(1, "b.n")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Get{
ID: 1,
Frame: "b.n",
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "intersect()" function.
func TestParser_Parse_Intersect(t *testing.T) {
q, err := pql.ParseString(`intersect(get(1), get(2))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Intersect{
Inputs: pql.BitmapCalls{
&pql.Get{ID: 1},
&pql.Get{ID: 2},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "range()" function with keyed args.
func TestParser_Parse_Range_Key(t *testing.T) {
q, err := pql.ParseString(`range(start="2000-01-02T03:04", id=20, frame="b.n", end="2001-01-02T03:04")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "range()" function with array args.
func TestParser_Parse_Range_Array(t *testing.T) {
q, err := pql.ParseString(`range(20, "b.n", "2000-01-02T03:04", "2001-01-02T03:04")`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Range{
ID: 20,
Frame: "b.n",
StartTime: time.Date(2000, 1, 2, 3, 4, 0, 0, time.UTC),
EndTime: time.Date(2001, 1, 2, 3, 4, 0, 0, time.UTC),
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "set()" function with keyed args.
func TestParser_Parse_Set_Key(t *testing.T) {
q, err := pql.ParseString(`set(id=1, frame="b.n", filter=2, profile_id = 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Set{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "set()" function with array args.
func TestParser_Parse_Set_Array(t *testing.T) {
q, err := pql.ParseString(`set(1, "b.n", 2, 3)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Set{
ID: 1,
Frame: "b.n",
Filter: 2,
ProfileID: 3,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "top-n()" function with keyed args.
func TestParser_Parse_TopN_Key(t *testing.T) {
q, err := pql.ParseString(`top-n(frame="b.n", n=2)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Frame: "b.n",
N: 2,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "top-n()" function with array args.
func TestParser_Parse_TopN_Array(t *testing.T) {
q, err := pql.ParseString(`top-n("b.n", 2)`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.TopN{
Frame: "b.n",
N: 2,
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}
// Ensure the parser can parse a "union()" function.
func TestParser_Parse_Union(t *testing.T) {
q, err := pql.ParseString(`union(get(1), get(2))`)
if err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(q, &pql.Query{
Root: &pql.Union{
Inputs: pql.BitmapCalls{
&pql.Get{ID: 1},
&pql.Get{ID: 2},
},
},
}) {
t.Fatalf("unexpected query: %s", spew.Sdump(q))
}
}

258
pql/scanner.go Normal file
View file

@ -0,0 +1,258 @@
package pql
import (
"bufio"
"bytes"
"io"
"unicode"
)
// Scanner represents a PQL lexical scanner.
type Scanner struct {
r io.RuneScanner
pos Pos
}
// NewScanner returns a new instance of Scanner.
func NewScanner(r io.Reader) *Scanner {
return &Scanner{r: bufio.NewReader(r)}
}
// Scan returns the next token and position from the underlying reader.
func (s *Scanner) Scan() (tok Token, pos Pos, lit string) {
pos = s.pos
// Read next code point.
ch := s.read()
// If we see whitespace then consume all contiguous whitespace.
// If we see a letter, or certain acceptable special characters, then consume
// as an ident or reserved word. If we see quotes, then scan as string.
if isWhitespace(ch) {
s.unread()
return s.scanWhitespace()
} else if isIdentFirstChar(ch) {
s.unread()
return s.scanIdent()
} else if isDigit(ch) {
s.unread()
return s.scanNumber()
} else if ch == '"' || ch == '\'' {
s.unread()
return s.scanString()
}
// Otherwise parse individual characters.
switch ch {
case eof:
tok = EOF
return
case '=':
tok = EQ
case ',':
tok = COMMA
case '(':
tok = LPAREN
case ')':
tok = RPAREN
case '[':
tok = LBRACK
case ']':
tok = RBRACK
default:
tok = ILLEGAL
}
lit = string(ch)
return
}
// read returns the next code point from the underlying reader and updates the pos.
func (s *Scanner) read() rune {
// Read next rune from underlying reader.
ch, _, err := s.r.ReadRune()
if err != nil {
return eof
}
// Update position information.
if ch == '\n' {
s.pos.Line++
s.pos.Char = 0
} else {
s.pos.Char++
}
return ch
}
// unread pushes the previously read rune back onto the reader.
func (s *Scanner) unread() {
if s.pos.Char == 0 {
s.pos.Line--
} else {
s.pos.Char--
}
s.r.UnreadRune()
}
// scanWhitespace consumes the current rune and all contiguous whitespace.
func (s *Scanner) scanWhitespace() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if ch == eof {
break
} else if !isWhitespace(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
return WS, pos, buf.String()
}
func (s *Scanner) scanIdent() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if ch == eof {
break
} else if !isIdentChar(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
lit = buf.String()
// If the literal matches a keyword then return that keyword.
if tok = Lookup(lit); tok != IDENT {
return tok, pos, lit
}
return IDENT, pos, lit
}
// scanNumber consumes consecutive integer digits.
func (s *Scanner) scanNumber() (tok Token, pos Pos, lit string) {
pos = s.pos
var buf bytes.Buffer
for {
ch := s.read()
if !isDigit(ch) {
s.unread()
break
}
buf.WriteRune(ch)
}
return NUMBER, pos, buf.String()
}
// scanString consumes a single-quoted or double-quoted string.
func (s *Scanner) scanString() (tok Token, pos Pos, lit string) {
pos = s.pos
// This must be either a single- or double-quote.
ending := s.read()
var buf bytes.Buffer
for {
ch := s.read()
if ch == ending {
break
} else if ch == '\n' || ch == eof {
return BADSTRING, pos, buf.String()
} else if ch == '\\' {
next := s.read()
if next == 'n' {
buf.WriteRune('\n')
} else if next == '\\' {
buf.WriteRune('\\')
} else if next == '"' {
buf.WriteRune('"')
} else if next == '\'' {
buf.WriteRune('\'')
} else {
return BADSTRING, pos, buf.String()
}
} else {
buf.WriteRune(ch)
}
}
return STRING, pos, buf.String()
}
// bufScanner represents a wrapper for scanner to add a buffer.
// It provides a fixed-length circular buffer that can be unread.
type bufScanner struct {
s *Scanner
i int // buffer index
n int // buffer size
buf [3]struct {
tok Token
pos Pos
lit string
}
}
// newBufScanner returns a new buffered scanner for a reader.
func newBufScanner(r io.Reader) *bufScanner {
return &bufScanner{s: NewScanner(r)}
}
// Scan reads the next token from the scanner.
func (s *bufScanner) Scan() (tok Token, pos Pos, lit string) {
// If we have unread tokens then read them off the buffer first.
if s.n > 0 {
s.n--
return s.curr()
}
// Move buffer position forward and save the token.
s.i = (s.i + 1) % len(s.buf)
buf := &s.buf[s.i]
buf.tok, buf.pos, buf.lit = s.s.Scan()
return s.curr()
}
// unscan pushes the previously token back onto the buffer.
func (s *bufScanner) unscan() { s.n++ }
// curr returns the last read token.
func (s *bufScanner) curr() (tok Token, pos Pos, lit string) {
buf := &s.buf[(s.i-s.n+len(s.buf))%len(s.buf)]
return buf.tok, buf.pos, buf.lit
}
// pos returns the current position.
func (s *bufScanner) pos() Pos {
_, pos, _ := s.curr()
return pos
}
// isWhitespace returns true if the rune a Unicode space character.
func isWhitespace(ch rune) bool { return unicode.IsSpace(ch) }
// isLetter returns true if the rune is a letter.
func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }
// isDigit returns true if the rune is a digit.
func isDigit(ch rune) bool { return (ch >= '0' && ch <= '9') }
// isIdentChar returns true if the rune can be used in an unquoted identifier.
func isIdentChar(ch rune) bool { return isLetter(ch) || isDigit(ch) || ch == '_' || ch == '-' }
// isIdentFirstChar returns true if the rune can be used as the first char in an identifier.
func isIdentFirstChar(ch rune) bool { return isLetter(ch) }
const eof = rune(0)

49
pql/scanner_test.go Normal file
View file

@ -0,0 +1,49 @@
package pql_test
import (
"strings"
"testing"
"github.com/umbel/pilosa/pql"
)
func TestScanner_Scan(t *testing.T) {
var tests = []struct {
s string
tok pql.Token
lit string
pos pql.Pos
}{
// Special tokens (EOF, ILLEGAL, WS)
{s: ``, tok: pql.EOF},
{s: `#`, tok: pql.ILLEGAL, lit: `#`},
{s: ` `, tok: pql.WS, lit: " "},
{s: "\t", tok: pql.WS, lit: "\t"},
{s: "\n", tok: pql.WS, lit: "\n"},
{s: `=`, tok: pql.EQ, lit: `=`},
{s: `,`, tok: pql.COMMA, lit: `,`},
{s: `(`, tok: pql.LPAREN, lit: `(`},
{s: `)`, tok: pql.RPAREN, lit: `)`},
{s: `[`, tok: pql.LBRACK, lit: `[`},
{s: `]`, tok: pql.RBRACK, lit: `]`},
{s: `foo`, tok: pql.IDENT, lit: `foo`},
{s: `100`, tok: pql.NUMBER, lit: `100`},
{s: `all`, tok: pql.ALL, lit: `all`},
{s: `ALL`, tok: pql.ALL, lit: `ALL`}, // case insensitive
}
for i, tt := range tests {
s := pql.NewScanner(strings.NewReader(tt.s))
tok, pos, lit := s.Scan()
if tt.tok != tok {
t.Errorf("%d. %q token mismatch: exp=%q got=%q <%q>", i, tt.s, tt.tok, tok, lit)
} else if tt.pos.Line != pos.Line || tt.pos.Char != pos.Char {
t.Errorf("%d. %q pos mismatch: exp=%#v got=%#v", i, tt.s, tt.pos, pos)
} else if tt.lit != lit {
t.Errorf("%d. %q literal mismatch: exp=%q got=%q", i, tt.s, tt.lit, lit)
}
}
}

81
pql/token.go Normal file
View file

@ -0,0 +1,81 @@
package pql
import "strings"
// Token is a lexical token of the PQL language.
type Token int
const (
// Special tokens
ILLEGAL Token = iota
EOF
WS
literal_beg
IDENT // main
STRING // "foo"
BADSTRING // bad escape or unclosed string
NUMBER // 12345
literal_end
keyword_beg
ALL
keyword_end
EQ // =
COMMA // ,
LPAREN // (
RPAREN // )
LBRACK // (
RBRACK // )
)
var tokens = [...]string{
ILLEGAL: "ILLEGAL",
EOF: "EOF",
WS: "WS",
IDENT: "IDENT",
NUMBER: "NUMBER",
ALL: "ALL",
EQ: "=",
COMMA: ",",
LPAREN: "(",
RPAREN: ")",
LBRACK: "(",
RBRACK: ")",
}
var keywords map[string]Token
func init() {
keywords = make(map[string]Token)
for tok := keyword_beg + 1; tok < keyword_end; tok++ {
keywords[strings.ToLower(tokens[tok])] = tok
}
}
// String returns the string representation of the token.
func (tok Token) String() string {
if tok >= 0 && tok < Token(len(tokens)) {
return tokens[tok]
}
return ""
}
// Lookup returns the token associated with a given string.
func Lookup(ident string) Token {
if tok, ok := keywords[strings.ToLower(ident)]; ok {
return tok
}
return IDENT
}
// Pos specifies the line and character position of a token.
// The Char and Line are both zero-based indexes.
type Pos struct {
Line int
Char int
}

348
query/lexer.go
View file

@ -1,348 +0,0 @@
package query
import (
"errors"
"fmt"
"strings"
"unicode"
"unicode/utf8"
log "github.com/cihub/seelog"
)
const (
TYPE_FUNC = iota
TYPE_LP = iota
TYPE_RP = iota
TYPE_LB = iota
TYPE_RB = iota
TYPE_VALUE = iota
TYPE_KEYWORD = iota
TYPE_EQUALS = iota
TYPE_COMMA = iota
TYPE_ERROR = iota
// Below types deprecated
TYPE_ID = iota
TYPE_FRAME = iota
TYPE_PROFILE = iota
TYPE_LIMIT = iota
)
type Token struct {
Text string
Type int
}
type statefn func(lexer *Lexer) statefn
type Lexer struct {
text string // the string being scanned.
pos int // current position in the input.
width int // width of last rune read from input.
start int // start position of this item.
state int // current state of lexer NEEDED???
ch chan Token // channel of scanned items (Tokens).
}
func (lexer *Lexer) emit(typ int) {
log.Trace("Lexer.emit", typ)
if lexer.start < lexer.pos {
lexer.ch <- Token{lexer.text[lexer.start:lexer.pos], typ}
}
lexer.start = lexer.pos
}
func (lexer *Lexer) acceptUntil(chars string, consume bool) (rune, error) {
log.Trace("Lexer.acceptUntil", chars, consume)
start_pos := lexer.pos
for {
next := lexer.next()
if next == rune(' ') {
lexer.ignore()
}
if next == 0 {
lexer.pos = start_pos
return 0, errors.New("Not found")
}
ch := strings.IndexRune(chars, next)
if ch >= 0 {
if consume {
lexer.backup()
} else {
lexer.pos = start_pos
}
return []rune(chars)[ch], nil
}
}
}
func (lexer *Lexer) acceptRun(valid string) {
log.Trace("Lexer.acceptRun", valid)
for strings.IndexRune(valid, lexer.next()) >= 0 {
}
lexer.backup()
}
// next returns the next rune in the input.
func (lexer *Lexer) next() (runey rune) {
log.Trace("Lexer.next", runey)
if lexer.pos >= len(lexer.text) {
lexer.width = 0
return 0
}
runey, lexer.width = utf8.DecodeRuneInString(lexer.text[lexer.pos:])
lexer.pos += lexer.width
return runey
}
// ignore skips over the pending input before this point.
func (lexer *Lexer) ignore() {
log.Trace("Lexer.ignore")
lexer.start = lexer.pos
}
// backup steps back one rune.
// Can be called only once per call of next.
func (lexer *Lexer) backup() {
log.Trace("Lexer.backup")
lexer.pos -= lexer.width
}
// peek returns but does not consume
// the next rune in the input.
func (lexer *Lexer) peek() rune {
log.Trace("Lexer.peek")
for {
next_rune := lexer.next()
// ignore spaces
if next_rune != rune(' ') {
lexer.backup()
return next_rune
}
lexer.ignore()
}
}
func stateError(err error) func(lexer *Lexer) statefn {
log.Trace("stateError", err)
return func(lexer *Lexer) statefn {
lexer.ch <- Token{err.Error(), TYPE_ERROR}
close(lexer.ch)
return nil
}
}
func stateFunc(lexer *Lexer) statefn {
log.Trace("stateFunc", lexer)
_, err := lexer.acceptUntil("(", true)
if err != nil {
return stateError(err)
}
lexer.emit(TYPE_FUNC)
return stateLP
}
func stateLP(lexer *Lexer) statefn {
log.Trace("stateLP", lexer)
lexer.pos += 1
lexer.emit(TYPE_LP)
// handle multiple LPs
if lexer.peek() == rune('(') {
return stateLP
}
return stateArgs
}
func stateLB(lexer *Lexer) statefn {
log.Trace("stateLB", lexer)
lexer.acceptUntil("[", true)
lexer.next()
lexer.emit(TYPE_LB)
peeked := lexer.peek()
if peeked == rune(']') {
lexer.next()
lexer.emit(TYPE_RB)
return stateArgs
} else if unicode.IsDigit(peeked) {
for {
r, err := lexer.acceptUntil(",]", true)
if err != nil {
return stateError(errors.New("Unclosed bracket!"))
}
lexer.emit(TYPE_VALUE)
if r == ',' {
lexer.next()
lexer.emit(TYPE_COMMA)
} else {
lexer.next()
lexer.emit(TYPE_RB)
return stateArgs
}
}
} else {
return stateArgs
}
}
func stateRB(lexer *Lexer) statefn {
log.Trace("stateRB", lexer)
lexer.pos += 1
lexer.emit(TYPE_RB)
peeked := lexer.peek()
if peeked == rune(',') {
return stateRPComma
} else if peeked == rune(')') {
return stateRP
} else {
return stateEOF
}
}
func stateArgs(lexer *Lexer) statefn {
log.Trace("stateArgs", lexer)
r, err := lexer.acceptUntil("(),=[]", false)
if err != nil {
return stateError(err)
}
switch r {
case '(':
return stateFunc
case ')':
return stateValue
case ',':
return stateValue
case '=':
return stateKeyword
case '[':
return stateLB
case ']':
return stateRB
default:
return stateError(errors.New("Expecting arguments!"))
}
}
func stateKeyword(lexer *Lexer) statefn {
log.Trace("stateKeyword", lexer)
_, err := lexer.acceptUntil("=", true)
if err != nil {
return stateError(err)
}
lexer.emit(TYPE_KEYWORD)
return stateEquals
}
func stateEquals(lexer *Lexer) statefn {
log.Trace("stateEquals", lexer)
e := lexer.next()
if e != '=' {
return stateError(errors.New("Expecting '='!"))
}
lexer.emit(TYPE_EQUALS)
return stateValue
}
func stateValue(lexer *Lexer) statefn {
log.Trace("stateValue", lexer)
r, err := lexer.acceptUntil("(),[", false)
if err != nil {
return stateError(err)
}
switch r {
case '(':
return stateFunc
case ')':
lexer.acceptUntil(")", true)
if lexer.pos > lexer.start {
lexer.emit(TYPE_VALUE)
}
return stateRP
case ',':
lexer.acceptUntil(",", true)
lexer.emit(TYPE_VALUE)
return stateComma
case '[':
return stateLB
default:
return stateError(errors.New("Unexpected character!"))
}
}
func stateRP(lexer *Lexer) statefn {
log.Trace("stateRP", lexer)
lexer.pos += 1
lexer.emit(TYPE_RP)
peeked := lexer.peek()
if peeked == rune(',') {
return stateRPComma
} else if peeked == rune(')') {
return stateRP
} else if peeked == rune(']') {
return stateRB
} else {
return stateEOF
}
}
func stateRPComma(lexer *Lexer) statefn {
log.Trace("stateRPComma", lexer)
lexer.pos += 1
lexer.emit(TYPE_COMMA)
return stateValue
}
func stateComma(lexer *Lexer) statefn {
log.Trace("stateComma", lexer)
lexer.pos += 1
lexer.emit(TYPE_COMMA)
return stateArgs
}
func stateEOF(lexer *Lexer) statefn {
log.Trace("stateEOF", lexer)
close(lexer.ch)
return nil
}
func (lexer *Lexer) Lex() (tokens []Token, err error) {
log.Trace("Lexer.Lex", tokens, err)
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("query: %v", r)
}
}
}()
tokens = make([]Token, 0)
state := stateFunc
go func() {
for {
state = state(lexer)
if state == nil {
return
}
}
}()
for t := range lexer.ch {
if t.Type == TYPE_ERROR {
err = errors.New(t.Text)
}
tokens = append(tokens, t)
}
return tokens, err
}
func Lex(input string) ([]Token, error) {
log.Trace("Lex", input)
lexer := Lexer{input, 0, 0, 0, TYPE_FUNC, make(chan Token)}
return lexer.Lex()
}

279
query/lexer_test.go
View file

@ -1,279 +0,0 @@
package query_test
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/query"
)
// Ensure a simple function and value can be lexed.
func TestLexer_Lex_FuncValue(t *testing.T) {
if tokens, err := query.Lex("get(10)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure a simple function, keyword, & value can be lexed.
func TestLexer_Lex_FuncKeywordValue(t *testing.T) {
if tokens, err := query.Lex("get(id=10)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"id", query.TYPE_KEYWORD},
{"=", query.TYPE_EQUALS},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure a more complex function can be lexed.
func TestLexer_Lex_FuncComplex(t *testing.T) {
if tokens, err := query.Lex("get(id=10, frame=brand)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"id", query.TYPE_KEYWORD},
{"=", query.TYPE_EQUALS},
{"10", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"frame", query.TYPE_KEYWORD},
{"=", query.TYPE_EQUALS},
{"brand", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that nested functions can be lexed.
func TestLexer_Lex_FuncNested(t *testing.T) {
if tokens, err := query.Lex("union(get(10))"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"union", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a list of nested functions can be lexed.
func TestLexer_Lex_FuncNestedList(t *testing.T) {
if tokens, err := query.Lex("intersect(get(10), get(11), get(12))"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"intersect", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"11", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"12", query.TYPE_VALUE},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that complex nested functions can be lexed.
func TestLexer_Lex_FuncNestedComplex(t *testing.T) {
if tokens, err := query.Lex("intersect(get(10), get(11), concat(get(12),get(14)))"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"intersect", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"11", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"concat", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"12", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"14", query.TYPE_VALUE},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that complex nested functions can be lexed.
func TestLexer_Lex_FuncNestedComplex2(t *testing.T) {
if tokens, err := query.Lex("concat(get(1, brand),get(2))"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"concat", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"1", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"brand", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"2", query.TYPE_VALUE},
{")", query.TYPE_RP},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a set function can be lexed.
func TestLexer_Lex_Set1(t *testing.T) {
if tokens, err := query.Lex("set(1, 987)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"set", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"1", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"987", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a set function can be lexed.
func TestLexer_Lex_Set2(t *testing.T) {
if tokens, err := query.Lex("set(1, general, 987)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"set", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"1", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"general", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"987", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a TopN function can be lexed.
func TestLexer_Lex_TopN1(t *testing.T) {
if tokens, err := query.Lex("top-n(get(10), 8)"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"top-n", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"8", query.TYPE_VALUE},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a TopN function can be lexed.
func TestLexer_Lex_TopN2(t *testing.T) {
if tokens, err := query.Lex("top-n(get(10, general), [1,2,3])"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"top-n", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"general", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"[", query.TYPE_LB},
{"1", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"2", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"3", query.TYPE_VALUE},
{"]", query.TYPE_RB},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// Ensure that a plugin function can be lexed.
func TestLexer_Lex_Plugin(t *testing.T) {
if tokens, err := query.Lex("plugin(get(10, general), [get(11, general)])"); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(tokens, []query.Token{
{"plugin", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"10", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"general", query.TYPE_VALUE},
{")", query.TYPE_RP},
{",", query.TYPE_COMMA},
{"[", query.TYPE_LB},
{"get", query.TYPE_FUNC},
{"(", query.TYPE_LP},
{"11", query.TYPE_VALUE},
{",", query.TYPE_COMMA},
{"general", query.TYPE_VALUE},
{")", query.TYPE_RP},
{"]", query.TYPE_RB},
{")", query.TYPE_RP},
}) {
t.Fatalf("unexpected tokens:\n\n%s", spew.Sprint(tokens))
}
}
// MustLex lexes s and returns a set of tokens. Panic on error.
func MustLex(s string) []query.Token {
a, err := query.Lex(s)
if err != nil {
panic(err)
}
return a
}

354
query/parser.go
View file

@ -1,354 +0,0 @@
package query
import (
"errors"
"fmt"
"strconv"
"time"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
)
var InvalidQueryError = errors.New("Invalid query format.")
type QueryParser struct {
tokens []Token
pos int
}
func (self *QueryParser) next() *Token {
log.Trace("QueryParser.next")
self.pos += 1
if self.pos > len(self.tokens) {
return nil
}
return &self.tokens[self.pos-1]
}
func (self *QueryParser) peek() *Token {
log.Trace("QueryParser.peek")
token := self.next()
self.backup()
return token
}
func (self *QueryParser) backup() {
log.Trace("QueryParser.backup")
self.pos -= 1
}
func (self *QueryParser) Parse() (query *Query, err error) {
log.Trace("QueryParser.Parse", query, err)
defer func() {
if r := recover(); r != nil {
var ok bool
err, ok = r.(error)
if !ok {
err = fmt.Errorf("query: %v", r)
}
}
}()
var token *Token
id := pilosa.NewGUID()
query = &Query{Id: &id, Subqueries: make([]Query, 0), Args: make(map[string]interface{})}
token = self.next()
if token.Type != TYPE_FUNC {
return nil, fmt.Errorf("Expected function, found token %v.", token)
}
query.Operation = token.Text
token = self.next()
if token.Type != TYPE_LP {
return nil, fmt.Errorf("Expected '(', found token %v.", token)
}
const shortForm = "2006-01-02T15:04"
ArgLoop:
for {
token = self.next()
if token == nil {
return nil, fmt.Errorf("Unclosed parentheses!")
}
switch token.Type {
case TYPE_FUNC:
self.backup()
subquery, err := self.Parse()
if err != nil {
return nil, err
}
query.Subqueries = append(query.Subqueries, *subquery)
case TYPE_VALUE:
switch query.Operation {
case "range":
switch len(query.Args) {
case 0:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["id"] = i
case 1:
query.Args["frame"] = token.Text
case 2:
t, err := time.Parse(shortForm, token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer DateTime (%v)", err)
}
query.Args["start"] = t
case 3:
t, err := time.Parse(shortForm, token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer DateTime (%v)", err)
}
query.Args["end"] = t
}
case "mask":
switch len(query.Args) {
case 0:
query.Args["frame"] = token.Text
case 1:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["start"] = i
case 2:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["end"] = i
}
case "get":
switch len(query.Args) {
case 0:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["id"] = i
case 1:
query.Args["frame"] = token.Text
default:
return nil, fmt.Errorf("Unexpected argument! (%v)", token)
}
case "clear":
switch len(query.Args) {
case 0:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["id"] = i
case 1:
query.Args["frame"] = token.Text
case 2:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["filter"] = i
case 3:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["profile_id"] = i
default:
return nil, fmt.Errorf("Unexpected argument! (%v)", token)
}
case "set":
switch len(query.Args) {
case 0:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["id"] = i
case 1:
query.Args["frame"] = token.Text
case 2:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["filter"] = i
case 3:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["profile_id"] = i
default:
return nil, fmt.Errorf("Unexpected argument! (%v)", token)
}
case "top-n":
switch len(query.Args) {
case 0:
query.Args["frame"] = token.Text
case 1:
i, err := strconv.Atoi(token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer! (%v)", err)
}
query.Args["n"] = i
}
case "all":
// do nothing
case "recall": //need pair based list of frag_id,handle
arg, ok := query.Args["stash"]
if !ok {
arg = &Stash{make([]CacheItem, 0), false}
query.Args["stash"] = arg
}
recallArgs := arg.(*Stash)
if !recallArgs.incomplete {
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting fragment id! (%v)", err)
}
recallArgs.Add(pilosa.SUUID(i)) //constructs a new arg
} else {
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting handle! (%v)", err)
}
recallArgs.Assign(pilosa.BitmapHandle(i)) //sets the value of the last created arg
}
default:
spew.Dump("UNPROCESSED VALUE", token)
}
continue
case TYPE_COMMA:
continue
case TYPE_RP:
break ArgLoop
case TYPE_KEYWORD:
var value interface{}
keyword := token.Text
token = self.next()
if token == nil || token.Type != TYPE_EQUALS {
return nil, fmt.Errorf("Expecting equals sign!")
}
token = self.next()
if token == nil || token.Type != TYPE_VALUE {
return nil, fmt.Errorf("Expecting value!")
}
if keyword == "id" {
value, err = strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
} else if keyword == "n" {
value, err = strconv.Atoi(token.Text)
if err != nil {
return nil, fmt.Errorf("Expecting integer! (%v)", err)
}
} else {
value = token.Text
}
query.Args[keyword] = value
case TYPE_LB:
peeked := self.peek()
// we currently support 2 types of values in square brackets:
// ids <- list of integers (TYPE_VALUE)
// filters <- list of queries (TYPE_FUNC)
switch peeked.Type {
case TYPE_VALUE:
query.Args["ids"] = make([]uint64, 0)
for {
token = self.next()
if token == nil {
return nil, fmt.Errorf("Unclosed list!")
}
switch token.Type {
case TYPE_COMMA:
break
case TYPE_VALUE:
i, err := strconv.ParseUint(token.Text, 10, 64)
if err != nil {
return nil, fmt.Errorf("Expecting integer id! (%v)", err)
}
query.Args["ids"] = append(query.Args["ids"].([]uint64), i)
case TYPE_RB:
continue ArgLoop
default:
return nil, fmt.Errorf("Unexpected token! (%v)", token)
}
}
case TYPE_FUNC:
query.Args["filters"] = make([]Query, 0)
for {
token = self.next()
if token == nil {
return nil, fmt.Errorf("Unclosed list!")
}
switch token.Type {
case TYPE_COMMA:
break
case TYPE_FUNC:
self.backup()
filterquery, err := self.Parse()
if err != nil {
return nil, err
}
query.Args["filters"] = append(query.Args["filters"].([]Query), *filterquery)
case TYPE_RB:
continue ArgLoop
default:
return nil, fmt.Errorf("Unexpected token! (%v)", token)
}
}
}
case TYPE_RB:
//
default:
log.Warn(spew.Sdump("unexpected", token))
return nil, errors.New("BAD TOKEN")
}
}
if query.Operation == "get" && query.Args["frame"] == nil {
query.Args["frame"] = "general"
}
if len(query.Args) == 0 && len(query.Subqueries) == 0 {
if query.Operation == "count" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "intersect" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "union" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "difference" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "range" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "mask" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "stash" {
return nil, fmt.Errorf("No Args Given")
}
if query.Operation == "recall" {
return nil, fmt.Errorf("No Args Given")
}
}
return query, nil
}
func Parse(tokens []Token) (*Query, error) {
log.Trace("Parse", tokens)
parser := QueryParser{tokens, 0}
return parser.Parse()
}

157
query/parser_test.go
View file

@ -1,157 +0,0 @@
package query_test
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/query"
)
// Ensure the parser can parse a get() query.
func TestParser_Parse_Get(t *testing.T) {
if q, err := query.Parse(MustLex("get(10)")); err != nil {
t.Fatal(err)
} else if q.Operation != "get" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{
"id": uint64(10),
"frame": "general",
}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a get() query with a keyword.
func TestParser_Parse_Get_Keyword(t *testing.T) {
if q, err := query.Parse(MustLex("get(id=10)")); err != nil {
t.Fatal(err)
} else if q.Operation != "get" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{"id": uint64(10), "frame": "general"}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a get() query with a keyword.
func TestParser_Parse_Get_MultiKeyword(t *testing.T) {
if q, err := query.Parse(MustLex("get(id=10, frame=brands)")); err != nil {
t.Fatal(err)
} else if q.Operation != "get" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{"id": uint64(10), "frame": "brands"}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a clear() query.
func TestParser_Parse_Clear(t *testing.T) {
if q, err := query.Parse(MustLex("clear(10, general, 0, 20)")); err != nil {
t.Fatal(err)
} else if q.Operation != "clear" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{
"id": uint64(10),
"frame": "general",
"filter": uint64(0),
"profile_id": uint64(20),
}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a set() query.
func TestParser_Parse_Set(t *testing.T) {
if q, err := query.Parse(MustLex("set(10, general, 0, 20)")); err != nil {
t.Fatal(err)
} else if q.Operation != "set" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{
"id": uint64(10),
"frame": "general",
"filter": uint64(0),
"profile_id": uint64(20),
}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse a nested query.
func TestParser_Parse_Nested(t *testing.T) {
q, err := query.Parse(MustLex("union(get(10,general), get(11,brand), get(12))"))
if err != nil {
t.Fatal(err)
} else if q.Operation != "union" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if len(q.Subqueries) != 3 {
t.Fatalf("unexpected subquery count: %d", len(q.Subqueries))
}
if sq := q.Subqueries[0]; sq.Operation != "get" {
t.Fatalf("unexpected subquery(0) operation: %q", sq.Operation)
} else if !reflect.DeepEqual(sq.Args, map[string]interface{}{"id": uint64(10), "frame": "general"}) {
t.Fatalf("unexpected subquery(0) args:\n\n%s", spew.Sprint(q.Args))
}
if sq := q.Subqueries[1]; sq.Operation != "get" {
t.Fatalf("unexpected subquery(1) operation: %q", sq.Operation)
} else if !reflect.DeepEqual(sq.Args, map[string]interface{}{"id": uint64(11), "frame": "brand"}) {
t.Fatalf("unexpected subquery(1) args:\n\n%s", spew.Sprint(sq.Args))
}
if sq := q.Subqueries[2]; sq.Operation != "get" {
t.Fatalf("unexpected subquery(2) operation: %q", sq.Operation)
} else if !reflect.DeepEqual(sq.Args, map[string]interface{}{"id": uint64(12), "frame": "general"}) {
t.Fatalf("unexpected subquery(2) args:\n\n%s", spew.Sprint(sq.Args))
}
}
// Ensure the parser can parse a query with lists.
func TestParser_Parse_Lists(t *testing.T) {
q, err := query.Parse(MustLex("top-n(get(10, general), [1,2,3], 50)"))
if err != nil {
t.Fatal(err)
} else if q.Operation != "top-n" {
t.Fatalf("unexpected operation: %q", q.Operation)
} else if !reflect.DeepEqual(q.Args, map[string]interface{}{"ids": []uint64{1, 2, 3}, "n": 50}) {
t.Fatalf("unexpected args:\n\n%s", spew.Sprint(q.Args))
}
if sq := q.Subqueries[0]; sq.Operation != "get" {
t.Fatalf("unexpected subquery(0) operation: %q", sq.Operation)
} else if !reflect.DeepEqual(sq.Args, map[string]interface{}{"id": uint64(10), "frame": "general"}) {
t.Fatalf("unexpected subquery(0) args:\n\n%s", spew.Sprint(q.Args))
}
}
// Ensure the parser can parse "all()".
func TestParser_Parse_All(t *testing.T) {
q, err := query.Parse(MustLex("top-n(all(), general, 30)"))
if err != nil {
t.Fatal(err)
} else if q.Operation != "top-n" {
t.Fatalf("unexpected operation: %q", q.Operation)
}
if sq := q.Subqueries[0]; sq.Operation != "all" {
t.Fatalf("unexpected subquery(0) operation: %q", sq.Operation)
}
}
// Ensure the parser can parse bracketed lists.
func TestParser_Parse_Lists_Bracketed(t *testing.T) {
q, err := query.Parse(MustLex("plugin(get(99), [get(10), get(11)])"))
if err != nil {
t.Fatalf("expected error")
}
spew.Dump(q)
}
// Ensure the parser can parse a recall query.
func TestParser_Parse_Recall(t *testing.T) {
q, err := query.Parse(MustLex("recall(12345,1,12345,2,12345,3)"))
if err != nil {
t.Fatal(err)
}
spew.Dump(q)
}

953
query/planner.go
View file

@ -1,953 +0,0 @@
package query
import (
"encoding/gob"
"errors"
"fmt"
"math/rand"
"time"
log "github.com/cihub/seelog"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
type PortableQueryStep interface {
GetId() *pilosa.GUID
GetLocation() *db.Location
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// ERRORS
///////////////////////////////////////////////////////////////////////////////////////////////////
// Invalid Frame
type InvalidFrame struct {
Db string
Frame string
Retry bool
}
func NewInvalidFrame(db string, frame string) *InvalidFrame {
return &InvalidFrame{db, frame, false}
}
func (self *InvalidFrame) Error() string {
return fmt.Sprintf("Invalid Frame: %s:%s", self.Db, self.Frame)
}
// Fragment Not Found
type FragmentNotFound struct {
Db string
Frame string
Slice int
Retry bool
}
func NewFragmentNotFound(db string, frame string, slice int) *FragmentNotFound {
return &FragmentNotFound{db, frame, slice, true}
}
func (self *FragmentNotFound) Error() string {
return fmt.Sprintf("Fragment Not Found: %s:%s:%d", self.Db, self.Frame, self.Slice)
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// BASE
///////////////////////////////////////////////////////////////////////////////////////////////////
type BaseQueryStep struct {
Id *pilosa.GUID
Operation string
Location *db.Location
Destination *db.Location
}
func (self *BaseQueryStep) GetId() *pilosa.GUID {
log.Trace("BaseQueryStep", *self.Id)
return self.Id
}
func (self *BaseQueryStep) GetLocation() *db.Location {
log.Trace("BaseQueryStep.GetLocation", *self.Location)
return self.Location
}
func (self *BaseQueryStep) LocIsDest() bool {
log.Trace("BaseQueryStep.LocIsDest")
if self.Location.ProcessId.Equals(self.Destination.ProcessId) &&
self.Location.FragmentId == self.Destination.FragmentId {
log.Trace("BaseQueryStep.LocIsDest Return true")
return true
}
log.Trace("BaseQueryStep.LocIsDest Return false")
return false
}
type BaseQueryResult struct {
Id *pilosa.GUID
Data interface{}
}
func (self *BaseQueryResult) ResultId() *pilosa.GUID {
log.Trace("BaseQueryStep.ResultId", self)
return self.Id
}
func (self *BaseQueryResult) ResultData() interface{} {
log.Trace("BaseQueryStep.ResultData", self)
return self.Data
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// COUNT
///////////////////////////////////////////////////////////////////////////////////////////////////
type CountQueryStep struct {
*BaseQueryStep
Input *pilosa.GUID
}
type CountQueryResult struct {
*BaseQueryResult
}
// QueryTree for COUNT queries
type CountQueryTree struct {
subquery QueryTree
location *db.Location
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *CountQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("CountQueryTree.getLocation", d, qt)
return qt.subquery.getLocation(d)
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// TOP-N
///////////////////////////////////////////////////////////////////////////////////////////////////
type TopNQueryStep struct {
*BaseQueryStep
Input *pilosa.GUID
Filters []uint64
N int
Frame string
}
type TopNQueryResult struct {
*BaseQueryResult
}
// QueryTree for TOP-N queries
type TopNQueryTree struct {
subquery QueryTree
location *db.Location
Filters []uint64
N int
Frame string
Slice int
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *TopNQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("TopNQueryTree.getLocation", d, qt)
var err error
if qt.location == nil {
frame := d.GetOrCreateFrame(qt.Frame)
slice := d.GetOrCreateSlice(qt.Slice)
fragment, err := d.GetFragmentForFrameSlice(frame, slice)
if err != nil {
log.Warn("GetFragmentForFrameSliceFailed TopNQueryTree", frame, slice)
return nil, err
}
qt.location = fragment.GetLocation()
return qt.location, nil
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// UNION
///////////////////////////////////////////////////////////////////////////////////////////////////
type UnionQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
}
type UnionQueryResult struct {
*BaseQueryResult
}
// QueryTree for UNION queries
type UnionQueryTree struct {
subqueries []QueryTree
location *db.Location
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *UnionQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("UnionQueryTree.getLocation", d, qt)
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// INTERSECT
///////////////////////////////////////////////////////////////////////////////////////////////////
type IntersectQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
}
type IntersectQueryResult struct {
*BaseQueryResult
}
// QueryTree for UNION queries
type IntersectQueryTree struct {
subqueries []QueryTree
location *db.Location
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *IntersectQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("IntersectQueryTree.getLocation", d, qt)
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// DIFFERENCE
///////////////////////////////////////////////////////////////////////////////////////////////////
type DifferenceQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
}
type DifferenceQueryResult struct {
*BaseQueryResult
}
// QueryTree for UNION queries
type DifferenceQueryTree struct {
subqueries []QueryTree
location *db.Location
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *DifferenceQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("DifferenceQueryTree.getLocation", d, qt)
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// CAT
///////////////////////////////////////////////////////////////////////////////////////////////////
type CatQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
N int
}
type Appendable interface {
Append(subq QueryTree)
}
type CatQueryResult struct {
*BaseQueryResult
}
// QueryTree for CAT queries
type CatQueryTree struct {
subqueries []QueryTree
location *db.Location
N int
}
func (qt *CatQueryTree) Append(subtree QueryTree) {
log.Trace("CatQueryTree.Append", qt, subtree)
qt.subqueries = append(qt.subqueries, subtree)
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *CatQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("CatQueryTree.getLocation", d)
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// ALL
///////////////////////////////////////////////////////////////////////////////////////////////////
// QueryTree for GET queries
type AllQueryTree struct {
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *AllQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("AllQueryTree.getLocation", d)
return nil, nil
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// GET
///////////////////////////////////////////////////////////////////////////////////////////////////
type GetQueryStep struct {
*BaseQueryStep
Bitmap *db.Bitmap
Slice int
}
type GetQueryResult struct {
*BaseQueryResult
}
// QueryTree for GET queries
type GetQueryTree struct {
bitmap *db.Bitmap
slice int
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *GetQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("GetQueryTree.getLocation", d)
slice := d.GetOrCreateSlice(qt.slice) // TODO: this should probably be just GetSlice (no create)
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("GetFragmenForBitmapFailed GetQueryTree", slice)
return nil, err
}
return fragment.GetLocation(), nil
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// SET
///////////////////////////////////////////////////////////////////////////////////////////////////
type SetQueryStep struct {
*BaseQueryStep
Bitmap *db.Bitmap
ProfileId uint64
}
type SetQueryResult struct {
*BaseQueryResult
}
// QueryTree for SET queries
type SetQueryTree struct {
bitmap *db.Bitmap
profile_id uint64
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *SetQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("SetQueryTree.getLocation", d)
// check here for supported frames
if !d.IsValidFrame(qt.bitmap.FrameType) {
return nil, NewInvalidFrame(d.Name, qt.bitmap.FrameType)
}
slice, err := d.GetSliceForProfile(qt.profile_id)
if err != nil {
return nil, NewFragmentNotFound(d.Name, qt.bitmap.FrameType, db.GetSlice(qt.profile_id))
}
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("NOT FOUND:", slice, qt.bitmap)
return nil, NewFragmentNotFound(d.Name, qt.bitmap.FrameType, db.GetSlice(qt.profile_id))
}
return fragment.GetLocation(), nil
}
///////////////////////////////////////////////////////////////////////////////////////////////////
func init() {
gob.Register(BaseQueryResult{})
gob.Register(SetQueryResult{})
gob.Register(ClearQueryResult{})
gob.Register(GetQueryResult{})
gob.Register(RangeQueryResult{})
gob.Register(CatQueryResult{})
gob.Register(UnionQueryResult{})
gob.Register(IntersectQueryResult{})
gob.Register(DifferenceQueryResult{})
gob.Register(CountQueryResult{})
gob.Register(TopNQueryResult{})
gob.Register(FillResult{})
gob.Register(StashQueryResult{})
gob.Register(CacheItem{})
gob.Register(Stash{})
gob.Register(SetQueryStep{})
gob.Register(ClearQueryStep{})
gob.Register(GetQueryStep{})
gob.Register(RangeQueryStep{})
gob.Register(CatQueryStep{})
gob.Register(UnionQueryStep{})
gob.Register(IntersectQueryStep{})
gob.Register(DifferenceQueryStep{})
gob.Register(CountQueryStep{})
gob.Register(TopNQueryStep{})
gob.Register(StashQueryStep{})
}
///////////////////////////////////////////////////////////////////////////////////////////////////
// This is the output of the query planner. Contains a list of steps which can be performed in parallel
//type QueryPlan []QueryStep
type QueryPlan []interface{}
type QueryPlanner struct {
Database *db.Database
Query *Query
}
type QueryTree interface {
getLocation(d *db.Database) (*db.Location, error)
}
func validateRange(Args map[string]interface{}) error {
log.Trace("validateRange", Args)
_, ok := Args["id"]
if !ok {
return errors.New("missing bitmap id")
}
_, ok = Args["frame"]
if !ok {
return errors.New("missing frame ")
}
_, ok = Args["start"]
if !ok {
return errors.New("missing start time")
}
_, ok = Args["end"]
if !ok {
return errors.New("missing end time")
}
return nil
}
// Builds QueryTree object from Query. Pass slice=-1 to perform operation on all slices
func (self *QueryPlanner) buildTree(query *Query, slice int) (QueryTree, error) {
log.Trace("QueryPlanner.buildTree", query, slice)
var tree QueryTree
// handle SET operation regardless of the slice
if query.Operation == "set" {
tree = &SetQueryTree{&db.Bitmap{Id: query.Args["id"].(uint64), FrameType: query.Args["frame"].(string), Filter: query.Args["filter"].(uint64)}, query.Args["profile_id"].(uint64)}
return tree, nil
}
if query.Operation == "clear" {
tree = &ClearQueryTree{&db.Bitmap{Id: query.Args["id"].(uint64), FrameType: query.Args["frame"].(string), Filter: query.Args["filter"].(uint64)}, query.Args["profile_id"].(uint64)}
return tree, nil
}
if query.Operation == "recall" {
tree = &RecallQueryTree{query.Args["stash"].(Stash)}
return tree, nil
}
// handle the remaining operations, taking slice into consideration
//I'm kinda thinking for stash it needs a differnt handler..i guess for now I'll see if i can use the cathandler
if slice == -1 {
var n int
n_, ok := query.Args["n"]
if ok {
n = n_.(int)
}
var p Appendable
if query.Operation == "stash" {
tree = &StashQueryTree{N: n}
} else {
tree = &CatQueryTree{N: n}
}
p = tree.(Appendable)
slice_ids, err := self.Database.SliceIds()
if err != nil {
return nil, err
}
for i := range slice_ids {
subtree, err := self.buildTree(query, slice_ids[i])
if err != nil {
return nil, err
}
p.Append(subtree)
}
} else {
if query.Operation == "get" {
tree = &GetQueryTree{&db.Bitmap{Id: query.Args["id"].(uint64), FrameType: query.Args["frame"].(string), Filter: 0}, slice}
return tree, nil
} else if query.Operation == "range" {
err := validateRange(query.Args)
if err != nil {
return nil, err
}
tree = &RangeQueryTree{
&db.Bitmap{
Id: query.Args["id"].(uint64),
FrameType: query.Args["frame"].(string),
Filter: 0},
slice,
query.Args["start"].(time.Time),
query.Args["end"].(time.Time)}
return tree, nil
} else if query.Operation == "count" {
subquery, err := self.buildTree(&query.Subqueries[0], slice)
if err != nil {
return nil, err
}
tree = &CountQueryTree{subquery: subquery}
} else if query.Operation == "all" {
tree = &AllQueryTree{}
} else if query.Operation == "top-n" {
var frame string
var n int
var filters []uint64
frame_, ok := query.Args["frame"]
if ok {
frame = frame_.(string)
}
n_, ok := query.Args["n"]
if ok {
n = n_.(int)
}
filters_, ok := query.Args["ids"]
if ok {
filters = filters_.([]uint64)
}
subquery, err := self.buildTree(&query.Subqueries[0], slice)
if err != nil {
return nil, err
}
tree = &TopNQueryTree{subquery: subquery, Filters: filters, N: n, Frame: frame, Slice: slice}
} else if query.Operation == "union" {
subqueries := make([]QueryTree, len(query.Subqueries))
var err error
for i, query := range query.Subqueries {
subqueries[i], err = self.buildTree(&query, slice)
if err != nil {
return nil, err
}
}
tree = &UnionQueryTree{subqueries: subqueries}
} else if query.Operation == "intersect" {
subqueries := make([]QueryTree, len(query.Subqueries))
var err error
for i, query := range query.Subqueries {
subqueries[i], err = self.buildTree(&query, slice)
if err != nil {
return nil, err
}
}
tree = &IntersectQueryTree{subqueries: subqueries}
} else if query.Operation == "difference" {
subqueries := make([]QueryTree, len(query.Subqueries))
var err error
for i, query := range query.Subqueries {
subqueries[i], err = self.buildTree(&query, slice)
if err != nil {
return nil, err
}
}
tree = &DifferenceQueryTree{subqueries: subqueries}
} else if query.Operation == "stash" {
subqueries := make([]QueryTree, len(query.Subqueries))
var err error
for i, query := range query.Subqueries {
subqueries[i], err = self.buildTree(&query, slice)
if err != nil {
return nil, err
}
}
tree = &StashQueryTree{subqueries: subqueries}
} else {
//TODO return error gracefully
log.Warn(spew.Sdump(query))
return nil, errors.New("BuildTree Issues")
}
}
return tree, nil
}
// Produces flattened QueryPlan from QueryTree input
func (self *QueryPlanner) flatten(qt QueryTree, id *pilosa.GUID, location *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanner.flatten", self, qt, id, location)
plan := QueryPlan{}
if cat, ok := qt.(*CatQueryTree); ok {
inputs := make([]*pilosa.GUID, len(cat.subqueries))
loc, err := cat.getLocation(self.Database)
if err != nil {
return nil, err
}
step := CatQueryStep{&BaseQueryStep{id, "cat", loc, location}, inputs, cat.N}
for index, subq := range cat.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if stash, ok := qt.(*StashQueryTree); ok {
inputs := make([]*pilosa.GUID, len(stash.subqueries))
loc, err := stash.getLocation(self.Database)
if err != nil {
return nil, err
}
step := StashQueryStep{&BaseQueryStep{id, "stash", loc, location}, inputs, stash.N}
for index, subq := range stash.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if union, ok := qt.(*UnionQueryTree); ok {
inputs := make([]*pilosa.GUID, len(union.subqueries))
loc, err := union.getLocation(self.Database)
if err != nil {
return nil, err
}
step := UnionQueryStep{&BaseQueryStep{id, "union", loc, location}, inputs}
for index, subq := range union.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if intersect, ok := qt.(*IntersectQueryTree); ok {
inputs := make([]*pilosa.GUID, len(intersect.subqueries))
loc, err := intersect.getLocation(self.Database)
if err != nil {
return nil, err
}
step := IntersectQueryStep{&BaseQueryStep{id, "intersect", loc, location}, inputs}
for index, subq := range intersect.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if difference, ok := qt.(*DifferenceQueryTree); ok {
inputs := make([]*pilosa.GUID, len(difference.subqueries))
loc, err := difference.getLocation(self.Database)
if err != nil {
return nil, err
}
step := DifferenceQueryStep{&BaseQueryStep{id, "difference", loc, location}, inputs}
for index, subq := range difference.subqueries {
sub_id := pilosa.NewGUID()
step.Inputs[index] = &sub_id
subq_steps, err := self.flatten(subq, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
} else if get, ok := qt.(*GetQueryTree); ok {
loc, err := get.getLocation(self.Database)
if err != nil {
return nil, err
}
step := GetQueryStep{&BaseQueryStep{id, "get", loc, location}, get.bitmap, get.slice}
plan := QueryPlan{step}
return &plan, nil
} else if rang, ok := qt.(*RangeQueryTree); ok {
loc, err := rang.getLocation(self.Database)
if err != nil {
return nil, err
}
step := RangeQueryStep{&BaseQueryStep{id, "range", loc, location}, rang.bitmap, rang.start, rang.end}
plan := QueryPlan{step}
return &plan, nil
} else if set, ok := qt.(*SetQueryTree); ok {
loc, err := set.getLocation(self.Database)
if err != nil {
return nil, err
}
step := SetQueryStep{&BaseQueryStep{id, "set", loc, location}, set.bitmap, set.profile_id}
plan := QueryPlan{step}
return &plan, nil
} else if clear, ok := qt.(*ClearQueryTree); ok {
loc, err := clear.getLocation(self.Database)
if err != nil {
return nil, err
}
step := ClearQueryStep{&BaseQueryStep{id, "clear", loc, location}, clear.bitmap, clear.profile_id}
plan := QueryPlan{step}
return &plan, nil
} else if cnt, ok := qt.(*CountQueryTree); ok {
sub_id := pilosa.NewGUID()
loc, err := cnt.getLocation(self.Database)
if err != nil {
return nil, err
}
step := &CountQueryStep{&BaseQueryStep{id, "count", loc, location}, &sub_id}
subq_steps, err := self.flatten(cnt.subquery, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
plan = append(plan, step)
} else if topn, ok := qt.(*TopNQueryTree); ok {
sub_id := pilosa.NewGUID()
loc, err := topn.getLocation(self.Database)
if err != nil {
return nil, err
}
step := &TopNQueryStep{&BaseQueryStep{id, "top-n", loc, location}, nil, topn.Filters, topn.N, topn.Frame}
switch topn.subquery.(type) {
case *AllQueryTree:
// do nothing
default:
step.Input = &sub_id
subq_steps, err := self.flatten(topn.subquery, &sub_id, loc)
if err != nil {
return nil, err
}
plan = append(plan, *subq_steps...)
}
plan = append(plan, step)
}
return &plan, nil
}
// Transforms Query into QueryTree and flattens to QueryPlan object
func (self *QueryPlanner) Plan(query *Query, id *pilosa.GUID, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanner.Plan", self, query, id, destination)
queryTree, err := self.buildTree(query, -1)
if err != nil {
return nil, err
}
return self.flatten(queryTree, query.Id, destination)
}
///////////////////////////////////////////////////////////////////////////////////////////////////
//MASK
///////////////////////////////////////////////////////////////////////////////////////////////////
type MaskQueryStep struct {
*BaseQueryStep
start, end uint64
}
type MaskQueryResult struct {
*BaseQueryResult
}
// QueryTree for Mask queries
type MaskQueryTree struct {
start, end uint64
bitmap *db.Bitmap
}
// Uses consistent hashing function to select node containing data for GET operation
/*
func (qt *MaskQueryTree) getLocation(d *db.Database) (*db.Location, error) {
slice := d.GetOrCreateSlice(qt.slice) // TODO: this should probably be just GetSlice (no create)
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("GetFragmenForBitmapFailed GetQueryTree", slice)
return nil, err
}
return fragment.GetLocation(), nil
}
*/
///////////////////////////////////////////////////////////////////////////////////////////////////
//Range
///////////////////////////////////////////////////////////////////////////////////////////////////
type RangeQueryStep struct {
*BaseQueryStep
Bitmap *db.Bitmap
Start, End time.Time
}
type RangeQueryResult struct {
*BaseQueryResult
}
// QueryTree for Mask queries
type RangeQueryTree struct {
bitmap *db.Bitmap
slice int
start, end time.Time
}
func (qt *RangeQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("RangeQueryTree", qt, d)
slice := d.GetOrCreateSlice(qt.slice) // TODO: this should probably be just GetSlice (no create)
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("GetFragmenForBitmapFailed GetQueryTree", slice)
return nil, err
}
return fragment.GetLocation(), nil
}
type FillResult struct {
*BaseQueryResult
}
///////////////////////////////////////////////////////////////////////////////////////////////////
//Stash
///////////////////////////////////////////////////////////////////////////////////////////////////
type CacheItem struct {
FragmentId pilosa.SUUID
Handle pilosa.BitmapHandle
}
type Stash struct {
Stash []CacheItem //pilosa.BitmapHandle //probably need to make the a struct with fragment_id and handle
incomplete bool
}
func NewStash() Stash {
return Stash{make([]CacheItem, 0), false}
}
func (st *Stash) Add(i pilosa.SUUID) {
item := CacheItem{i, 0}
st.Stash = append(st.Stash, item)
st.incomplete = true
}
func (st *Stash) Assign(i pilosa.BitmapHandle) {
st.Stash[len(st.Stash)-1].Handle = i //big assumption that item already exists
st.incomplete = false
}
type StashQueryStep struct {
*BaseQueryStep
Inputs []*pilosa.GUID
N int
}
type StashQueryResult struct {
*BaseQueryResult
}
// QueryTree for UNION queries
type StashQueryTree struct {
subqueries []QueryTree
location *db.Location
N int
}
func (qt *StashQueryTree) Append(subtree QueryTree) {
qt.subqueries = append(qt.subqueries, subtree)
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *StashQueryTree) getLocation(d *db.Database) (*db.Location, error) {
var err error
if qt.location == nil {
subqueryLength := len(qt.subqueries)
if subqueryLength > 0 {
locationIndex := rand.Intn(subqueryLength)
subquery := qt.subqueries[locationIndex]
qt.location, err = subquery.getLocation(d)
}
}
return qt.location, err
}
type RecallQueryStep struct {
*BaseQueryStep
Stash Stash
}
type RecallQueryTree struct {
Stash Stash
}
func (rqt *RecallQueryTree) getLocation(d *db.Database) (*db.Location, error) {
return nil, nil
}
type RecallQueryResult struct {
*BaseQueryResult
}
type ClearQueryStep struct {
*BaseQueryStep
Bitmap *db.Bitmap
ProfileId uint64
}
type ClearQueryResult struct {
*BaseQueryResult
}
type ClearQueryTree struct {
bitmap *db.Bitmap
profile_id uint64
}
// Uses consistent hashing function to select node containing data for GET operation
func (qt *ClearQueryTree) getLocation(d *db.Database) (*db.Location, error) {
log.Trace("ClearQueryTree.getLocation", d)
// check here for supported frames
if !d.IsValidFrame(qt.bitmap.FrameType) {
return nil, NewInvalidFrame(d.Name, qt.bitmap.FrameType)
}
slice, err := d.GetSliceForProfile(qt.profile_id)
if err != nil {
return nil, NewFragmentNotFound(d.Name, qt.bitmap.FrameType, db.GetSlice(qt.profile_id))
}
fragment, err := d.GetFragmentForBitmap(slice, qt.bitmap)
if err != nil {
log.Warn("NOT FOUND:", slice, qt.bitmap)
return nil, NewFragmentNotFound(d.Name, qt.bitmap.FrameType, db.GetSlice(qt.profile_id))
}
return fragment.GetLocation(), nil
}

67
query/planner_example.go
View file

@ -1,67 +0,0 @@
package query
//package main
//
//import (
// "pilosa/query"
// "pilosa/core"
// "math/rand"
// "time"
// "log"
//)
//
//func main() {
// rand.Seed(time.Now().UnixNano())
// cluster := core.Cluster{Self:"192.168.1.100:1201"}
// database := cluster.AddDatabase("property49")
// frame := database.AddFrame("general")
// frame.AddSlice("192.168.1.100:1201", "192.168.1.100:1202", "192.168.1.100:1203")
// frame.AddSlice("192.168.1.101:1201", "192.168.1.101:1202", "192.168.1.101:1203")
// frame.AddSlice("192.168.1.102:1201", "192.168.1.102:1202", "192.168.1.102:1203")
// frame2 := database.AddFrame("brands")
// frame2.AddSlice("192.168.1.200:1201", "192.168.1.200:1202", "192.168.1.200:1203")
// frame2.AddSlice("192.168.1.201:1201", "192.168.1.201:1202", "192.168.1.201:1203")
// frame2.AddSlice("192.168.1.202:1201", "192.168.1.202:1202", "192.168.1.202:1203")
//
// //cluster.describe()
//
// //query := Query{"get", []QueryInput{Bitmap{"general", 10}}}
//
// //query := Query{"union", []QueryInput{
// // &Query{"get", []QueryInput{Bitmap{"general", 20}}},
// // &Query{"get", []QueryInput{Bitmap{"brands", 30}}},
// //}}
//
// //queryString = "union(bitmap(general, 33), bitmap(brands, 44))"
// //queryString := `["union", ["intersect", ["bitmap", "general", 33], ["bitmap", "brands", 44]], ["bitmap", "general", 55]]`
// //query := `count(union(bitmap(""), bitmap(brands, 55)))`
// //query := `setbit(bitmap(cats, 33), 75000)`
// queryString := `["union", ["intersect", ["bitmap", "general", 33], ["bitmap", "brands", 44]], ["bitmap", "general", 55]]`
// queryParser := query.QueryParser{queryString}
// queryParsed, err := queryParser.Parse()
// log.Println(queryParsed)
// if err != nil {
// log.Println("ERROR!", err)
// }
//
//// query := qp.Query{"inter", []qp.QueryInput{
//// &qp.Query{"union", []qp.QueryInput{
//// &qp.Query{"get", []qp.QueryInput{core.Bitmap{"general", 20}}},
//// &qp.Query{"get", []qp.QueryInput{core.Bitmap{"brands", 30}}},
//// }},
//// &qp.Query{"get", []qp.QueryInput{core.Bitmap{"general", 10}}},
//// }}
////
// planner := query.QueryPlanner{&cluster, database}
//
// id, _ := uuid.NewV4()
// dest := "1.2.3.4:1234"
// log.Println("Query id", id, "Dest", dest)
//
// queryplan := planner.Plan(queryParsed, id, dest, -1)
//
// for _, i := range *queryplan {
// log.Println(i)
// }
// //fmt.Println(queryplan)
//}

283
query/planner_test.go
View file

@ -1,283 +0,0 @@
package query_test
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/db"
"github.com/umbel/pilosa/query"
"github.com/umbel/pilosa/util"
)
// Ensure the query planner can plan a "get" query.
func TestQueryPlanner_Plan_Get(t *testing.T) {
q, err := query.QueryForPQL("get(10,default)")
if err != nil {
t.Fatal(err)
}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: q}
id := util.RandomUUID()
plan, err := planner.Plan(q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 3 {
t.Fatalf("unexpected plan length: %d", n)
}
if step := (*plan)[0].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if step.Slice != 0 {
t.Fatalf("unexpected step(0) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(0) bitmap: %s", spew.Sprint(step.Bitmap))
}
if step := (*plan)[1].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(1) operation: %s", step.Operation)
} else if step.Slice != 1 {
t.Fatalf("unexpected step(1) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(1) bitmap: %s", spew.Sprint(step.Bitmap))
}
if step := (*plan)[2].(query.CatQueryStep); step.Operation != "cat" {
t.Fatalf("unexpected step(2) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[0].(query.GetQueryStep).Id,
(*plan)[1].(query.GetQueryStep).Id,
}) {
t.Fatalf("unexpected step(2) inputs: %s", spew.Sprint(step.Inputs))
}
}
// Ensure the query planner can plan a "set" query.
func TestQueryPlanner_Plan_Set(t *testing.T) {
q, err := query.QueryForPQL("set(10, default, 0, 100)")
if err != nil {
t.Fatal(err)
}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: q}
id := util.RandomUUID()
plan, err := planner.Plan(q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 1 {
t.Fatalf("unexpected plan length: %d", n)
}
if step := (*plan)[0].(query.SetQueryStep); step.Operation != "set" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if step.ProfileId != 100 {
t.Fatalf("unexpected step(0) profile id: %d", step.ProfileId)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(0) bitmap: %s", spew.Sprint(step.Bitmap))
}
}
// Ensure the query planner can plan a "top-n" query.
func TestQueryPlanner_Plan_TopN(t *testing.T) {
q, err := query.QueryForPQL("top-n(get(10, default), default, 50,[1,2,3])")
if err != nil {
t.Fatal(err)
}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: q}
id := util.RandomUUID()
plan, err := planner.Plan(q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 5 {
t.Fatalf("unexpected plan length: %d", n)
}
if step := (*plan)[0].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(0) bitmap: %s", spew.Sprint(step.Bitmap))
}
if step := (*plan)[1].(*query.TopNQueryStep); step.Operation != "top-n" {
t.Fatalf("unexpected step(1) operation: %s", step.Operation)
} else if step.Input != (*plan)[0].(query.GetQueryStep).Id {
t.Fatalf("unexpected step(1) input: %d", step.Input)
} else if step.N != 50 {
t.Fatalf("unexpected step(1) n: %d", step.N)
} else if !reflect.DeepEqual(step.Filters, []uint64{1, 2, 3}) {
t.Fatalf("unexpected step(1) filters: %s", spew.Sprint(step.Filters))
}
if step := (*plan)[2].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(2) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(2) bitmap: %s", spew.Sprint(step.Bitmap))
}
if step := (*plan)[3].(*query.TopNQueryStep); step.Operation != "top-n" {
t.Fatalf("unexpected step(3) operation: %s", step.Operation)
} else if step.Input != (*plan)[2].(query.GetQueryStep).Id {
t.Fatalf("unexpected step(3) input: %d", step.Input)
} else if step.N != 50 {
t.Fatalf("unexpected step(3) n: %d", step.N)
}
}
// Ensure the query planner can plan a "top-n" all() query.
func TestQueryPlanner_Plan_TopN_All(t *testing.T) {
q, err := query.QueryForPQL("top-n(all(), default, 50, [1,2,3])")
if err != nil {
t.Fatal(err)
}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: q}
id := util.RandomUUID()
plan, err := planner.Plan(q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 3 {
t.Fatalf("unexpected plan length: %d", n)
}
if step := (*plan)[0].(*query.TopNQueryStep); step.Operation != "top-n" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if step.Input != nil {
t.Fatalf("unexpected step(0) input: %v", step.Input)
} else if step.N != 50 {
t.Fatalf("unexpected step(0) n: %d", step.N)
} else if !reflect.DeepEqual(step.Filters, []uint64{1, 2, 3}) {
t.Fatalf("unexpected step(0) filters: %s", spew.Sprint(step.Filters))
}
}
// Ensure the query planner can plan a "union" query.
func TestQueryPlanner_Plan_Union(t *testing.T) {
id1 := util.RandomUUID()
q1 := query.Query{Id: &id1, Operation: "get", Args: map[string]interface{}{"id": uint64(10), "frame": "default"}}
id2 := util.RandomUUID()
q2 := query.Query{Id: &id2, Operation: "get", Args: map[string]interface{}{"id": uint64(20), "frame": "default"}}
id3 := util.RandomUUID()
q := query.Query{Id: &id3, Operation: "union", Subqueries: []query.Query{q1, q2}}
d, frag1 := NewDefaultDB()
planner := query.QueryPlanner{Database: d, Query: &q}
id := util.RandomUUID()
plan, err := planner.Plan(&q, &id, frag1.GetLocation())
if err != nil {
t.Fatal(err)
}
if n := len(*plan); n != 7 {
t.Fatalf("unexpected plan size: %d", n)
}
// First step should be a "get" step.
if step := (*plan)[0].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(0) operation: %s", step.Operation)
} else if step.Slice != 0 {
t.Fatalf("unexpected step(0) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(0) bitmap: %s", spew.Sprint(step.Bitmap))
}
// Second step should also be a "get" step.
if step := (*plan)[1].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(1) operation: %s", step.Operation)
} else if step.Slice != 0 {
t.Fatalf("unexpected step(1) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 20, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(1) bitmap: %s", spew.Sprint(step.Bitmap))
}
// Third step should union the first two steps.
if step := (*plan)[2].(query.UnionQueryStep); step.Operation != "union" {
t.Fatalf("unexpected step(2) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[0].(query.GetQueryStep).Id,
(*plan)[1].(query.GetQueryStep).Id,
}) {
t.Fatalf("unexpected step(2) inputs: %s", spew.Sprint(step.Inputs))
}
// Fourth step should be a "get" step.
if step := (*plan)[3].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(3) operation: %s", step.Operation)
} else if step.Slice != 1 {
t.Fatalf("unexpected step(3) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 10, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(3) bitmap: %s", spew.Sprint(step.Bitmap))
}
// Fifth step should also be a "get" step.
if step := (*plan)[4].(query.GetQueryStep); step.Operation != "get" {
t.Fatalf("unexpected step(4) operation: %s", step.Operation)
} else if step.Slice != 1 {
t.Fatalf("unexpected step(4) slice: %d", step.Slice)
} else if !reflect.DeepEqual(step.Bitmap, &db.Bitmap{Id: 20, FrameType: "default", Filter: 0}) {
t.Fatalf("unexpected step(4) bitmap: %s", spew.Sprint(step.Bitmap))
}
// Sixth step should union the previous two steps.
if step := (*plan)[5].(query.UnionQueryStep); step.Operation != "union" {
t.Fatalf("unexpected step(5) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[3].(query.GetQueryStep).Id,
(*plan)[4].(query.GetQueryStep).Id,
}) {
t.Fatalf("unexpected step(5) inputs: %s", spew.Sprint(step.Inputs))
}
// Final step should concatenate the two union steps.
if step := (*plan)[6].(query.CatQueryStep); step.Operation != "cat" {
t.Fatalf("unexpected step(6) operation: %s", step.Operation)
} else if !reflect.DeepEqual(step.Inputs, []*pilosa.GUID{
(*plan)[2].(query.UnionQueryStep).Id,
(*plan)[5].(query.UnionQueryStep).Id,
}) {
t.Fatalf("unexpected step(6) inputs: %s", spew.Sprint(step.Inputs))
}
}
// NewDefaultDB returns a simple, initialized database and fragment.
func NewDefaultDB() (*db.Database, *db.Fragment) {
// Create an empty database
c := db.NewCluster()
d := c.GetOrCreateDatabase("main")
f := d.GetOrCreateFrame("default")
s1 := d.GetOrCreateSlice(0)
frag1 := d.GetOrCreateFragment(f, s1, util.Id())
pid := util.RandomUUID()
p1 := db.NewProcess(&pid)
p1.SetHost("----192.1.1.0----")
frag1.SetProcess(p1)
s2 := d.GetOrCreateSlice(1)
frag2 := d.GetOrCreateFragment(f, s2, util.Id())
pid = util.RandomUUID()
p2 := db.NewProcess(&pid)
p2.SetHost("----192.1.1.1----")
frag2.SetProcess(p2)
return d, frag1
}

111
query/query.go
View file

@ -1,111 +0,0 @@
package query
import (
"strings"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/db"
)
type QueryInput interface{}
type QueryResults struct {
Data interface{}
}
type PqlList []PqlListItem
type PqlListItem struct {
Id *pilosa.GUID
Label string
PQL string
}
type Query struct {
Id *pilosa.GUID
Operation string
Args map[string]interface{}
Subqueries []Query
}
func QueryPlanForPQL(database *db.Database, pql string, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanFOrPQL", database, pql, destination)
tokens, err := Lex(pql)
if err != nil {
return nil, err
}
return QueryPlanForTokens(database, tokens, destination)
}
func QueryForPQL(pql string) (*Query, error) {
log.Trace("QueryForPQL", pql)
tokens, err := Lex(pql)
if err != nil {
return nil, err
}
return QueryForTokens(tokens)
}
func QueryForTokens(tokens []Token) (*Query, error) {
log.Trace("QueryForTokens", tokens)
query, err := Parse(tokens)
if err != nil {
return nil, err
}
return query, nil
}
func QueryPlanForTokens(database *db.Database, tokens []Token, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanForTokens", database, tokens, destination)
query, err := QueryForTokens(tokens)
if err != nil {
return nil, err
}
return QueryPlanForQuery(database, query, destination)
}
func QueryPlanForQuery(database *db.Database, query *Query, destination *db.Location) (*QueryPlan, error) {
log.Trace("QueryPlanForQuery", database, query, destination)
query_planner := QueryPlanner{Database: database, Query: query}
id := pilosa.NewGUID()
query_plan, err := query_planner.Plan(query, &id, destination)
if err != nil {
return nil, err
}
return query_plan, nil
}
func TokensToFilterStrings(tokens []Token) (string, []string) {
log.Trace("TokensToFilterStrings", tokens)
var whole []string
var filter string
var filters []string
var open_parens int
var in_square_brackets bool
var last_slice = 0
open_parens = -1
in_square_brackets = false
for i, _ := range tokens {
whole = append(whole, tokens[i].Text)
if tokens[i].Type == TYPE_FUNC {
last_slice = i
} else if tokens[i].Type == TYPE_LP {
open_parens += 1
} else if tokens[i].Type == TYPE_RP {
open_parens -= 1
if open_parens == 0 {
if !in_square_brackets {
filter = strings.Join(whole[2:], "")
} else {
filters = append(filters, strings.Join(whole[last_slice:], ""))
}
last_slice = i
}
} else if open_parens == 0 && tokens[i].Type == TYPE_LB {
in_square_brackets = true
}
}
return filter, filters
}

36
query/query_test.go
View file

@ -1,36 +0,0 @@
package query_test
import (
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/umbel/pilosa/query"
)
func TestTokensToFilterStrings1(t *testing.T) {
filter, filters := query.TokensToFilterStrings(MustLex("plugin(get(88, general), [get(12, general), get(13, general)])"))
if filter != "get(88,general)" {
t.Fatalf("unexpected filter: %s", filter)
} else if !reflect.DeepEqual(filters, []string{"get(12,general)", "get(13,general)"}) {
t.Fatalf("unexpected filters: %s", spew.Sprint(filters))
}
}
func TestTokensToFilterStrings2(t *testing.T) {
filter, filters := query.TokensToFilterStrings(MustLex("plugin(intersect(get(88, general, [0]), get(77, b.n)), [get(12, general), get(13, general)])"))
if filter != "intersect(get(88,general,[0]),get(77,b.n))" {
t.Fatalf("unexpected filter: %s", filter)
} else if !reflect.DeepEqual(filters, []string{"get(12,general)", "get(13,general)"}) {
t.Fatalf("unexpected filters: %s", spew.Sprint(filters))
}
}
func TestTokensToFilterStrings3(t *testing.T) {
filter, filters := query.TokensToFilterStrings(MustLex("plugin(intersect(get(88, general, [0]), get(77, b.n)))"))
if filter != "intersect(get(88,general,[0]),get(77,b.n))" {
t.Fatalf("unexpected filter: %s", filter)
} else if !reflect.DeepEqual(filters, []string(nil)) {
t.Fatalf("unexpected filters: %s", spew.Sprint(filters))
}
}

65
statsd/statd.go
View file

@ -1,65 +0,0 @@
package statsd
import (
"time"
"github.com/cactus/go-statsd-client/statsd"
log "github.com/cihub/seelog"
)
// DefaultHost is the default host to send statsd data to.
const DefaultHost = "127.0.0.1:8125"
// Host is the host to send statsd data to.
var Host = DefaultHost
type args struct {
stat string
delta int64
rate float32
}
var timer chan args
var count chan string
var end chan bool
func Setup() {
timer = make(chan args, 32768)
count = make(chan string, 32768)
end = make(chan bool)
log.Warn("New Stats", Host)
stats, _ := statsd.New(Host, "")
go func() {
for {
select {
case ci := <-timer:
stats.Gauge(ci.stat, ci.delta, ci.rate)
stats.Timing(ci.stat, ci.delta, ci.rate)
case stat := <-count:
stats.Inc(stat, 1, 1.0)
case <-end:
log.Warn("DONE Stats")
return
}
}
}()
}
func SendTimer(stat string, delta int64) {
pstat := "pilosa." + stat
milli := time.Duration(delta) / time.Millisecond
timer <- args{pstat, int64(milli), 1.0}
}
func SendInc(stat string) {
pstat := "pilosa." + stat
count <- pstat
}
func ShutdownStats() {
log.Warn("Shutdown Stats")
end <- true
}

49
storage.go
View file

@ -1,49 +0,0 @@
package pilosa
// Storage represents
type Storage interface {
Open() error
Close() error
Flush()
Fetch(bitmap_id uint64, db, frame string, slice int) (*Bitmap, uint64)
Store(id uint64, db, frame string, slice int, filter uint64, bitmap *Bitmap) error
StoreBlock(id uint64, db, frame string, slice int, filter uint64, chunk uint64, block_index int32, block uint64) error
RemoveBlock(id uint64, db, frame string, slice int, chunk uint64, block_index int32)
StoreBit(bid uint64, db, frame string, slice int, filter uint64, chunk uint64, block_index int32, block, count uint64)
RemoveBit(id uint64, db, frame string, slice int, filter uint64, chunk uint64, block_index int32, count uint64)
}
var storageFns = make(map[string]NewStorageFunc)
// NewStorageFunc represents a function that instantiates a new storage engine.
type NewStorageFunc func(opt StorageOptions) Storage
// RegisterStorage registers a storage engine.
func RegisterStorage(name string, fn NewStorageFunc) {
if storageFns[name] != nil {
panic("storage engine already registered: " + name)
}
storageFns[name] = fn
}
// NewStorage returns a new Storage instance by name.
func NewStorage(name string, opt StorageOptions) Storage {
fn := storageFns[name]
if fn == nil {
panic("storage type not registered: " + name)
}
return fn(opt)
}
// StorageOptions represents the options passed to the storage engine.
type StorageOptions struct {
DB string
Slice int
Frame string
FragmentID SUUID
LevelDBPath string
}

244
storage/cassandra/cassandra.go
View file

@ -1,244 +0,0 @@
package cassandra
import (
"time"
log "github.com/cihub/seelog"
"github.com/gocql/gocql"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
)
func init() {
pilosa.RegisterStorage("cassandra",
func(opt pilosa.StorageOptions) pilosa.Storage {
return NewStorage(opt)
},
)
}
// CREATE KEYSPACE IF NOT EXISTS pilosa WITH strategy_class = SimpleStrategy AND strategy_options:replication_factor = 1"
// create keyspace if not exists pilosa with replication = {'class': 'SimpleStrategy', 'replication_factor' : 1} and durable_writes = true;
// CREATE KEYSPACE pilosa WITH replication = {'class': 'NetworkTopologyStrategy', 'pilpang': '2'} AND durable_writes = true;
// CREATE TABLE IF NOT EXISTS bitmap (bitmap_id bigint, db varchar, frame varchar, slice int, filter int, chunkkey bigint, blockindex int, block bigint, PRIMARY KEY ((bitmap_id, db, frame, slice), chunkkey, blockindex) )
// DefaultHosts are the default hosts in the cassandra cluster.
var DefaultHosts = []string{"localhost"}
const (
// DefaultKeyspace is the default keyspace used in cassandra.
DefaultKeyspace = "pilosa"
// DefaultFlushInterval is the default maximum time between flushes.
DefaultFlushInterval = 5 * time.Second
// DefaultFlushThreshold is the default maximum number of items to batch.
DefaultFlushThreshold = 15
)
// Storage represents Cassandra-backed storage for bitmaps.
type Storage struct {
session *gocql.Session
batch *gocql.Batch
batchTime time.Time
batchN int
FlushInterval time.Duration
FlushThreshold int
Hosts []string
Keyspace string
}
// NewStorage returns a new, uninitialized instance of Storage.
func NewStorage(opt pilosa.StorageOptions) *Storage {
return &Storage{
batchTime: time.Now(),
FlushInterval: DefaultFlushInterval,
FlushThreshold: DefaultFlushThreshold,
Hosts: DefaultHosts,
Keyspace: DefaultKeyspace,
}
}
// Open opens the connection to the cassandra cluster.
func (s *Storage) Open() error {
// Create cluster configuration.
config := gocql.NewCluster(s.Hosts...)
config.Keyspace = s.Keyspace
config.Consistency = gocql.One
config.Timeout = 5 * time.Second
config.RetryPolicy = &gocql.SimpleRetryPolicy{NumRetries: 10}
// Connect to cassandra.
session, err := config.CreateSession()
if err != nil {
return err
}
s.session = session
return nil
}
// Close closes the connection to the cassandra cluster.
func (s *Storage) Close() error {
if s.session != nil {
s.Flush()
s.session.Close()
}
return nil
}
// Fetch returns a bitmap by ID.
func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*pilosa.Bitmap, uint64) {
bm := pilosa.NewBitmap()
// Start benchmark.
start := time.Now()
// Create iterator over bitmap.
itr := s.session.Query("SELECT filter, Chunkkey, BlockIndex, block FROM bitmap WHERE bitmap_id=? AND db=? AND frame=? AND slice=? ",
u64toi64(bitmapID), db, frame, slice).Iter()
// Iterate over chunks and materialize bitmap object.
var chunk *pilosa.Chunk
var chunkKey, block, count int64
var blockIndex uint32
var filter int
lastKey := int64(-1)
for itr.Scan(&filter, &chunkKey, &blockIndex, &block) {
if chunkKey != int64(-1) {
if chunkKey != lastKey {
chunk = &pilosa.Chunk{uint64(chunkKey), pilosa.NewBlocks()}
bm.AddChunk(chunk)
}
chunk.Value[uint8(blockIndex)] = uint64(block)
} else {
count = block
}
lastKey = chunkKey
}
statsd.SendTimer("cassandra_storage_Fetch", time.Since(start).Nanoseconds())
statsd.SendInc("cassandra_storage_Read")
// Set total bits set.
bm.SetCount(uint64(count))
return bm, uint64(filter)
}
// beginBatch starts a batch if one is not already in progress.
func (s *Storage) beginBatch() {
if s.batch == nil {
s.batch = gocql.NewBatch(gocql.UnloggedBatch)
}
s.batchN++
}
// endBatch flushes a batch if one is in progress.
func (s *Storage) endBatch() {
if s.batch == nil {
return
}
start := time.Now()
s.Flush()
statsd.SendTimer("cassandra_storage_EndBatch", time.Since(start).Nanoseconds())
statsd.SendInc("cassandra_storage_Write")
}
// Flush flushes the current batch to storage.
func (s *Storage) Flush() {
start := time.Now()
// If batch exists then flush it.
if s.batch != nil {
if err := s.session.ExecuteBatch(s.batch); err != nil {
log.Warn("Batch ERROR: ", err)
}
}
// Clear batch and threshold time and count.
s.batch = nil
s.batchTime = time.Now()
s.batchN = 0
statsd.SendTimer("cassandra_storage_FlushBatch", time.Since(start).Nanoseconds())
}
// Store saves a bitmap to storage.
func (s *Storage) Store(id uint64, db string, frame string, slice int, filter uint64, bm *pilosa.Bitmap) error {
s.beginBatch()
for i := bm.ChunkIterator(); !i.Limit(); i = i.Next() {
var chunk = i.Item()
for idx, block := range chunk.Value {
if block != 0 {
s.StoreBlock(id, db, frame, slice, filter, chunk.Key, int32(idx), block)
}
}
}
s.StoreBlock(id, db, frame, slice, filter, pilosa.CounterMask, 0, bm.BitCount())
s.endBatch()
return nil
}
// StoreBlock saves a block to storage.
func (s *Storage) StoreBlock(bid uint64, db string, frame string, slice int, filter uint64, bchunk uint64, blockIndex int32, bblock uint64) error {
id := u64toi64(bid) // these functions ignore overflow
block := u64toi64(bblock)
chunk := u64toi64(bchunk)
if s.batch == nil {
s.beginBatch()
}
start := time.Now()
s.batch.Query(`INSERT INTO bitmap ( bitmap_id, db, frame, slice , filter, ChunkKey, BlockIndex, block) VALUES (?,?,?,?,?,?,?,?) USING timestamp ?;`,
id, db, frame, slice, int(filter), chunk, blockIndex, block, start.UnixNano())
statsd.SendTimer("cassandra_storage_StoreBlock", time.Since(start).Nanoseconds())
return nil
}
// RemoveBlock deletes a block from storage.
func (s *Storage) RemoveBlock(bid uint64, db string, frame string, slice int, bchunk uint64, blockIndex int32) {
log.Trace("RemoveBlock", bid, db, frame, slice, bchunk, blockIndex)
id := u64toi64(bid) //these fucntions ignore overflow
chunk := u64toi64(bchunk)
if s.batch == nil {
s.beginBatch()
}
startTime := time.Now()
s.batch.Query(`DELETE FROM bitmap USING TIMESTAMP ? WHERE bitmap_id=? AND db=? AND frame=? AND slice=? AND chunkkey=? AND blockindex=?`,
startTime.UnixNano(), id, db, frame, slice, chunk, blockIndex)
statsd.SendTimer("cassandra_storage_DeleteBlock", time.Since(startTime).Nanoseconds())
}
func (s *Storage) StoreBit(bid uint64, db string, frame string, slice int, filter uint64, chunk uint64, blockIndex int32, val, count uint64) {
s.beginBatch()
s.StoreBlock(bid, db, frame, slice, filter, chunk, blockIndex, val)
s.StoreBlock(bid, db, frame, slice, filter, pilosa.CounterMask, 0, count)
s.endBatch()
}
func (s *Storage) RemoveBit(id uint64, db string, frame string, slice int, filter uint64, chunk uint64, blockIndex int32, count uint64) {
log.Trace("RemoveBit", id, db, frame, slice, chunk, blockIndex)
s.beginBatch()
s.RemoveBlock(id, db, frame, slice, chunk, blockIndex)
s.StoreBlock(id, db, frame, slice, filter, pilosa.CounterMask, 0, count)
s.endBatch()
}
func u64toi64(v uint64) int64 { return int64(v) }

1
storage/cassandra/cassandra_test.go
View file

@ -1 +0,0 @@
package cassandra_test

222
storage/leveldb/leveldb.go
View file

@ -1,222 +0,0 @@
package leveldb
import (
"bytes"
"encoding/binary"
"path/filepath"
"strconv"
"time"
"github.com/syndtr/goleveldb/leveldb"
. "github.com/syndtr/goleveldb/leveldb/util"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/statsd"
)
func init() {
pilosa.RegisterStorage("leveldb",
func(opt pilosa.StorageOptions) pilosa.Storage {
return NewStorage(opt)
},
)
}
//go get github.com/syndtr/goleveldb/leveldb
// Storage represents a LevelDB-backed storage engine.
type Storage struct {
path string
db *leveldb.DB
batch *leveldb.Batch
batchTime time.Time
batchN int
}
// NewStorage returns a new instance of Storage.
func NewStorage(opt pilosa.StorageOptions) *Storage {
path := filepath.Join(
opt.LevelDBPath,
opt.DB,
strconv.Itoa(opt.Slice),
opt.Frame,
opt.FragmentID.String(),
)
return &Storage{path: path}
}
// Path returns the path the storage was initialized with.
func (s *Storage) Path() string { return s.path }
// Open opens and initializes the storage.
func (s *Storage) Open() error {
db, err := leveldb.OpenFile(s.path, nil)
if err != nil {
return err
}
s.db = db
s.batchTime = time.Now().Add(-time.Hour)
return nil
}
// Close flushes and closes the storage.
func (s *Storage) Close() error {
s.Flush()
return s.db.Close()
}
func (s *Storage) Fetch(bitmapID uint64, db string, frame string, slice int) (*pilosa.Bitmap, uint64) {
bm := pilosa.NewBitmap()
// Begin benchmark.
start := time.Now()
// Create an iterator on the database.
iter := s.db.NewIterator(&Range{
Start: marshalKey(bitmapID, 0, 0),
Limit: marshalKey(bitmapID+1, 0, 0),
}, nil)
defer iter.Release()
// Iterate over blocks in database and create bitmap.
var chunk *pilosa.Chunk
var filter, block, count uint64
lastKey := uint64(pilosa.CounterMask)
for iter.Next() {
_, key, idx := unmarshalKey(iter.Key())
block, filter = unmarshalValue(iter.Value())
if key != pilosa.CounterMask {
if key != lastKey {
chunk = &pilosa.Chunk{key, pilosa.NewBlocks()}
bm.AddChunk(chunk)
}
chunk.Value[idx] = block
} else {
count = block
}
lastKey = key
}
statsd.SendTimer("leveldb_storage_Fetch", time.Since(start).Nanoseconds())
// Set bit count on bitmap.
bm.SetCount(uint64(count))
return bm, uint64(filter)
}
// beginBatch starts a new batch if one is not already started.
func (s *Storage) beginBatch() {
if s.batch == nil {
s.batch = &leveldb.Batch{}
}
s.batchN++
}
func (s *Storage) endBatch() {
if s.batch == nil {
return
}
start := time.Now()
if time.Since(s.batchTime) > 15*time.Second || s.batchN > 300 {
s.Flush()
}
statsd.SendTimer("leveldb_storage_EndBatch", time.Since(start).Nanoseconds())
}
func (s *Storage) Flush() {
start := time.Now()
// Flush the batch if one exists.
if s.batch != nil {
s.db.Write(s.batch, nil)
}
// Clear batch and reset timer and count.
s.batch = nil
s.batchTime = time.Now()
s.batchN = 0
statsd.SendTimer("leveldb_storage_FlushBatch", time.Since(start).Nanoseconds())
}
// Store saves a bitmap to storage.
func (s *Storage) Store(bitmapID uint64, db string, frame string, slice int, filter uint64, bm *pilosa.Bitmap) error {
s.beginBatch()
for itr := bm.ChunkIterator(); !itr.Limit(); itr = itr.Next() {
for idx, block := range itr.Item().Value {
if block != 0 {
s.StoreBlock(bitmapID, db, frame, slice, filter, itr.Item().Key, int32(idx), block)
}
}
}
s.StoreBlock(bitmapID, db, frame, slice, filter, pilosa.CounterMask, 0, bm.BitCount())
s.endBatch()
return nil
}
// StoreBlock saves a block to storage.
func (s *Storage) StoreBlock(bitmapID uint64, db string, frame string, slice int, filter uint64, chunk uint64, index int32, block uint64) error {
start := time.Now()
s.batch.Put(marshalKey(bitmapID, chunk, uint8(index)), marshalValue(block, filter))
statsd.SendTimer("leveldb_storage_StoreBlock", time.Since(start).Nanoseconds())
return nil
}
// RemoveBlock deletes a block from storage. This is a no-op in the LevelDB store.
func (s *Storage) RemoveBlock(bitmapID uint64, db string, frame string, slice int, chunk uint64, blockIndex int32) {
}
// StoreBit sets a bit in storage.
func (s *Storage) StoreBit(bitmapID uint64, db string, frame string, slice int, filter uint64, bchunk uint64, blockIndex int32, bblock, count uint64) {
s.beginBatch()
s.StoreBlock(bitmapID, db, frame, slice, filter, bchunk, blockIndex, bblock)
s.StoreBlock(bitmapID, db, frame, slice, filter, pilosa.CounterMask, 0, count)
s.endBatch()
}
// RemoveBlock unsets a bit in storage. This is a no-op in the LevelDB store.
func (s *Storage) RemoveBit(bitmapID uint64, db string, frame string, slice int, filter uint64, bchunk uint64, blockIndex int32, count uint64) {
}
// marshalKey encodes the id, chunk key, & block index to a byte slice.
func marshalKey(id, key uint64, index uint8) []byte {
buf := new(bytes.Buffer)
binary.Write(buf, binary.LittleEndian, id)
binary.Write(buf, binary.LittleEndian, key)
binary.Write(buf, binary.LittleEndian, index)
return buf.Bytes()
}
// marshalValue encodes the block number and filter to a byte slice.
func marshalValue(block, filter uint64) []byte {
buf := new(bytes.Buffer)
binary.Write(buf, binary.LittleEndian, block)
binary.Write(buf, binary.LittleEndian, filter)
return buf.Bytes()
}
// unmarshalKey decodes the id, chunk key, & block index from a byte slice.
func unmarshalKey(v []byte) (id, key uint64, index uint8) {
buf := bytes.NewReader(v)
binary.Read(buf, binary.LittleEndian, &id)
binary.Read(buf, binary.LittleEndian, &key)
binary.Read(buf, binary.LittleEndian, &index)
return
}
// unmarshalValue decodes the block number and filter from a byte slice.
func unmarshalValue(v []byte) (block, filter uint64) {
buf := bytes.NewReader(v)
binary.Read(buf, binary.LittleEndian, &block)
binary.Read(buf, binary.LittleEndian, &filter)
return
}

55
storage/leveldb/leveldb_test.go
View file

@ -1,55 +0,0 @@
package leveldb_test
import (
"io/ioutil"
"os"
"testing"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/storage/leveldb"
"github.com/umbel/pilosa/util"
)
func init() {
util.SetupStatsd()
}
func TestStorage_Fetch(t *testing.T) {
s := MustOpenStorage()
defer s.Close()
}
// Storage represents a test wrapper for leveldb.Storage.
type Storage struct {
*leveldb.Storage
}
// NewStorage returns a new instance of Storage with a temporary path.
func NewStorage() *Storage {
// Create temporary path.
f, err := ioutil.TempFile("", "pilosa-leveldb-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return &Storage{leveldb.NewStorage(pilosa.StorageOptions{
LevelDBPath: f.Name(),
})}
}
// MustOpenStorage returns a new, opened instance of Storage.
func MustOpenStorage() *Storage {
s := NewStorage()
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the storage and removes the underlying data file.
func (s *Storage) Close() error {
defer os.Remove(s.Path())
return s.Storage.Close()
}

79
storage/mem/mem.go
View file

@ -1,79 +0,0 @@
package mem
import (
"fmt"
"github.com/umbel/pilosa"
)
func init() {
pilosa.RegisterStorage("memory",
func(opt pilosa.StorageOptions) pilosa.Storage {
return NewStorage()
},
)
}
// Storage represents in-memory bitmap storage.
type Storage struct {
db map[string]*pilosa.Bitmap
}
// NewStorage returns a new instance of Storage.
func NewStorage() *Storage {
return &Storage{
db: make(map[string]*pilosa.Bitmap),
}
}
// Open initializes the storage.
func (c *Storage) Open() error { return nil }
// Close shuts down the storage.
func (c *Storage) Close() error { return nil }
// FlushBatch flushes the batch to disk. This is a no-op for in-memory storage.
func (c *Storage) Flush() {}
// Fetch retrieves a bitmap by id.
func (c *Storage) Fetch(id uint64, db, frame string, slice int) (*pilosa.Bitmap, uint64) {
key := fmt.Sprintf("%d:%s:%s:%d", id, db, frame, slice)
// Find bitmap by key.
b, ok := c.db[key]
if ok {
return b, 0
}
// If the bitmap doesn't exist then create a new one.
b = pilosa.NewBitmap()
c.db[key] = b
return b, 0
}
// Store saves a bitmap to storage.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (c *Storage) Store(id uint64, db, frame string, slice int, filter uint64, b *pilosa.Bitmap) error {
return nil
}
// StoreBlock saves a block to storage.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (c *Storage) StoreBlock(id uint64, db, frame string, slice int, filter uint64, chunk_key uint64, block_index int32, block uint64) error {
return nil
}
// RemoveBlock deletes a block from bitmap.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (self *Storage) RemoveBlock(id uint64, db string, frame string, slice int, chunk uint64, block_index int32) {
}
// StoreBit sets a bit in a bitmap.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (self *Storage) StoreBit(id uint64, db string, frame string, slice int, filter uint64, bchunk uint64, block_index int32, bblock, count uint64) {
}
// RemoveBit unsets a bit in a bitmap.
// This is a no-op for in-memory storage because changes are stored in the cache.
func (self *Storage) RemoveBit(id uint64, db string, frame string, slice int, filter uint64, chunk uint64, block_index int32, count uint64) {
}

30
storage/mem/mem_test.go
View file

@ -1,30 +0,0 @@
package mem_test
import (
"testing"
"github.com/umbel/pilosa/storage/mem"
)
// Ensure a bitmap can be retrieved from storage.
func TestStorage_Fetch(t *testing.T) {
s := mem.NewStorage()
// Retrieving a non-existent bitmap should create a new one.
b, _ := s.Fetch(1, "d", "f", 0)
if b == nil {
t.Fatal("expected bitmap")
}
// Retrieve the bitmap again.
other, _ := s.Fetch(1, "d", "f", 0)
if b != other {
t.Fatal("expected same bitmap")
}
// Retrieving a different bitmap should return a different reference.
b2, _ := s.Fetch(2, "d", "f", 0)
if b == b2 {
t.Fatal("expected new bitmap")
}
}

7
storage/storage.go
View file

@ -1,7 +0,0 @@
package storage
import (
_ "github.com/umbel/pilosa/storage/cassandra"
_ "github.com/umbel/pilosa/storage/leveldb"
_ "github.com/umbel/pilosa/storage/mem"
)

5
timeframe.go
View file

@ -86,15 +86,18 @@ func NextMonth(start time.Time, end time.Time) bool {
nextMonth := start.AddDate(0, 1, 0)
return sameMonth(nextMonth, end) || end.After(nextMonth)
}
func sameDay(t1, t2 time.Time) bool {
y1, m1, d1 := t1.Date()
y2, m2, d2 := t2.Date()
return (y1 == y2) && (m1 == m2) && (d1 == d2)
}
func NextDay(start time.Time, end time.Time) bool {
nextDay := start.AddDate(0, 0, 1)
return sameDay(nextDay, end) || end.After(nextDay)
}
func GetRange(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 {
results, marker := upHill(start_time, end_time, tile_id)
r2 := downHill(marker, end_time, tile_id)
@ -142,8 +145,8 @@ func upHill(start_time time.Time, end_time time.Time, tile_id uint64) ([]uint64,
return results, time_iterator
}
func downHill(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 {
func downHill(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 {
var results []uint64
time_iterator := start_time
for time_iterator.Before(end_time) {

56
transport/http.go
View file

@ -1,56 +0,0 @@
package transport
import (
"log"
"github.com/umbel/pilosa/db"
)
const DefaultHTTPPort = 15001
type HttpTransport struct {
port int
outbox chan *db.Message
done chan int
}
func (trans *HttpTransport) Init() error {
log.Println("Bind to port", trans.port)
trans.done = make(chan int)
go trans.Loop()
return nil
}
func (trans *HttpTransport) Loop() {
var message *db.Message
for {
select {
case message = <-trans.outbox:
log.Println(message)
case <-trans.done:
return
}
}
}
func (trans *HttpTransport) Close() {
log.Println("Closing HTTP transport.")
trans.done <- 1
}
func (trans *HttpTransport) Send(node string, message *db.Message) error {
log.Println("Send", message, "to", node)
trans.outbox <- message
return nil
}
func (trans *HttpTransport) Receive() (*db.Message, error) {
return &db.Message{}, nil
}
func NewHttpTransport(port int) *HttpTransport {
trans := new(HttpTransport)
trans.port = port
trans.outbox = make(chan *db.Message, 10)
return trans
}

187
transport/tcp.go
View file

@ -1,187 +0,0 @@
package transport
import (
"encoding/gob"
"fmt"
"net"
"os"
"time"
notify "github.com/bitly/go-notify"
log "github.com/cihub/seelog"
"github.com/umbel/pilosa"
"github.com/umbel/pilosa/core"
"github.com/umbel/pilosa/db"
)
const DefaultTCPPort = 12001
type connection struct {
transport *TcpTransport
inbox chan *db.Message
outbox chan *db.Message
conn *net.Conn
process *pilosa.GUID
}
type newconnection struct {
id *pilosa.GUID
connection *connection
}
func init() {
gob.Register(pilosa.GUID{})
}
func (self *connection) manage() {
BeginManageConnection:
for {
if self.conn == nil {
process, err := self.transport.ProcessMap.GetProcess(self.process)
if err != nil {
log.Warn("transport/tcp: error getting process, retrying in 2 seconds... ", self.process, err)
time.Sleep(2 * time.Second)
continue
}
host_string := fmt.Sprintf("%s:%d", process.Host(), process.PortTcp())
conn, err := net.Dial("tcp", host_string)
if err != nil {
log.Warn("transport/tcp: error dialing: ", host_string, " Retrying in 2 seconds...")
time.Sleep(2 * time.Second)
continue
}
self.conn = &conn
go func() {
self.outbox <- &db.Message{Data: self.transport.ID.String()}
}()
}
encoder := gob.NewEncoder(*self.conn)
decoder := gob.NewDecoder(*self.conn)
var exit = make(chan int)
go func() {
for {
var mess *db.Message
err := decoder.Decode(&mess)
if err != nil {
log.Warn("transport/tcp: error decoding message: ", err.Error())
exit <- 1
return
}
self.inbox <- mess
}
}()
for {
select {
case message := <-self.outbox:
err := encoder.Encode(message)
if err != nil {
log.Warn(err.Error())
return
}
case message := <-self.inbox:
identifier, ok := message.Data.(pilosa.GUID)
if ok {
// message is connection registration; bypass inbox and register
self.process = &identifier
self.transport.reg <- &newconnection{&identifier, self}
} else {
self.transport.inbox <- message
}
case <-exit:
if self.process != nil {
self.conn = nil
continue BeginManageConnection
} else {
return
}
}
}
}
}
type TcpTransport struct {
inbox chan *db.Message
outbox chan db.Envelope
connections map[pilosa.GUID]*connection
reg chan *newconnection
ID pilosa.GUID
Port int
ProcessMap *core.ProcessMap
}
func NewTcpTransport(id pilosa.GUID) *TcpTransport {
return &TcpTransport{
inbox: make(chan *db.Message, 100),
outbox: make(chan db.Envelope, 100),
connections: make(map[pilosa.GUID]*connection),
reg: make(chan *newconnection),
ID: id,
Port: DefaultTCPPort,
}
}
func (self *TcpTransport) Run() {
log.Warn("Initializing TCP transport")
go self.listen()
for {
select {
case env := <-self.outbox:
con, ok := self.connections[*(env.Host)]
if !ok {
con = &connection{self, make(chan *db.Message, 100), make(chan *db.Message, 100), nil, env.Host}
go con.manage()
self.connections[*env.Host] = con
}
con.outbox <- env.Message
case nc := <-self.reg:
self.connections[*nc.id] = nc.connection
}
}
}
func (self *TcpTransport) listen() {
port_string := fmt.Sprintf(":%d", self.Port)
l, e := net.Listen("tcp", port_string)
if e != nil {
log.Critical("Cannot bind to port! ", self.Port)
os.Exit(-1)
}
for {
conn, err := l.Accept()
if err != nil {
log.Warn("Error accepting, trying again in 2 sec... ", err)
time.Sleep(2 * time.Second)
continue
}
go self.manage(&conn)
}
}
func (self *TcpTransport) manage(conn *net.Conn) {
con := &connection{self, make(chan *db.Message, 1024), make(chan *db.Message, 1024), conn, nil}
con.manage()
}
func (self *TcpTransport) Close() {
log.Warn("Shutting down TCP transport")
}
func (self *TcpTransport) Send(message *db.Message, host *pilosa.GUID) {
log.Trace("TcpTransport.Send", message, host)
envelope := db.Envelope{Message: message, Host: host}
notify.Post("outbox", &envelope)
self.outbox <- envelope
}
func (self *TcpTransport) Receive() *db.Message {
log.Trace("TcpTransport.Receive")
message := <-self.inbox
notify.Post("inbox", message)
return message
}
func (self *TcpTransport) Push(message *db.Message) {
self.inbox <- message
}

27
util.go
View file

@ -1,27 +0,0 @@
package pilosa
import (
"io"
"os"
"strings"
"github.com/kr/s3/s3util"
)
func openFile(s string) (io.ReadCloser, error) {
if isURL(s) {
return s3util.Open(s, nil)
}
return os.Open(s)
}
func createFile(s string) (io.WriteCloser, error) {
if isURL(s) {
return s3util.Create(s, nil, nil)
}
return os.Create(s)
}
func isURL(s string) bool {
return strings.HasPrefix(s, "http://") || strings.HasPrefix(s, "https://")
}

950
vendor/github.com/BurntSushi/toml/decode_test.go generated vendored
View file

@ -1,950 +0,0 @@
package toml
import (
"fmt"
"log"
"reflect"
"testing"
"time"
)
func init() {
log.SetFlags(0)
}
func TestDecodeSimple(t *testing.T) {
var testSimple = `
age = 250
andrew = "gallant"
kait = "brady"
now = 1987-07-05T05:45:00Z
yesOrNo = true
pi = 3.14
colors = [
["red", "green", "blue"],
["cyan", "magenta", "yellow", "black"],
]
[My.Cats]
plato = "cat 1"
cauchy = "cat 2"
`
type cats struct {
Plato string
Cauchy string
}
type simple struct {
Age int
Colors [][]string
Pi float64
YesOrNo bool
Now time.Time
Andrew string
Kait string
My map[string]cats
}
var val simple
_, err := Decode(testSimple, &val)
if err != nil {
t.Fatal(err)
}
now, err := time.Parse("2006-01-02T15:04:05", "1987-07-05T05:45:00")
if err != nil {
panic(err)
}
var answer = simple{
Age: 250,
Andrew: "gallant",
Kait: "brady",
Now: now,
YesOrNo: true,
Pi: 3.14,
Colors: [][]string{
{"red", "green", "blue"},
{"cyan", "magenta", "yellow", "black"},
},
My: map[string]cats{
"Cats": cats{Plato: "cat 1", Cauchy: "cat 2"},
},
}
if !reflect.DeepEqual(val, answer) {
t.Fatalf("Expected\n-----\n%#v\n-----\nbut got\n-----\n%#v\n",
answer, val)
}
}
func TestDecodeEmbedded(t *testing.T) {
type Dog struct{ Name string }
type Age int
tests := map[string]struct {
input string
decodeInto interface{}
wantDecoded interface{}
}{
"embedded struct": {
input: `Name = "milton"`,
decodeInto: &struct{ Dog }{},
wantDecoded: &struct{ Dog }{Dog{"milton"}},
},
"embedded non-nil pointer to struct": {
input: `Name = "milton"`,
decodeInto: &struct{ *Dog }{},
wantDecoded: &struct{ *Dog }{&Dog{"milton"}},
},
"embedded nil pointer to struct": {
input: ``,
decodeInto: &struct{ *Dog }{},
wantDecoded: &struct{ *Dog }{nil},
},
"embedded int": {
input: `Age = -5`,
decodeInto: &struct{ Age }{},
wantDecoded: &struct{ Age }{-5},
},
}
for label, test := range tests {
_, err := Decode(test.input, test.decodeInto)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(test.wantDecoded, test.decodeInto) {
t.Errorf("%s: want decoded == %+v, got %+v",
label, test.wantDecoded, test.decodeInto)
}
}
}
func TestTableArrays(t *testing.T) {
var tomlTableArrays = `
[[albums]]
name = "Born to Run"
[[albums.songs]]
name = "Jungleland"
[[albums.songs]]
name = "Meeting Across the River"
[[albums]]
name = "Born in the USA"
[[albums.songs]]
name = "Glory Days"
[[albums.songs]]
name = "Dancing in the Dark"
`
type Song struct {
Name string
}
type Album struct {
Name string
Songs []Song
}
type Music struct {
Albums []Album
}
expected := Music{[]Album{
{"Born to Run", []Song{{"Jungleland"}, {"Meeting Across the River"}}},
{"Born in the USA", []Song{{"Glory Days"}, {"Dancing in the Dark"}}},
}}
var got Music
if _, err := Decode(tomlTableArrays, &got); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(expected, got) {
t.Fatalf("\n%#v\n!=\n%#v\n", expected, got)
}
}
// Case insensitive matching tests.
// A bit more comprehensive than needed given the current implementation,
// but implementations change.
// Probably still missing demonstrations of some ugly corner cases regarding
// case insensitive matching and multiple fields.
func TestCase(t *testing.T) {
var caseToml = `
tOpString = "string"
tOpInt = 1
tOpFloat = 1.1
tOpBool = true
tOpdate = 2006-01-02T15:04:05Z
tOparray = [ "array" ]
Match = "i should be in Match only"
MatcH = "i should be in MatcH only"
once = "just once"
[nEst.eD]
nEstedString = "another string"
`
type InsensitiveEd struct {
NestedString string
}
type InsensitiveNest struct {
Ed InsensitiveEd
}
type Insensitive struct {
TopString string
TopInt int
TopFloat float64
TopBool bool
TopDate time.Time
TopArray []string
Match string
MatcH string
Once string
OncE string
Nest InsensitiveNest
}
tme, err := time.Parse(time.RFC3339, time.RFC3339[:len(time.RFC3339)-5])
if err != nil {
panic(err)
}
expected := Insensitive{
TopString: "string",
TopInt: 1,
TopFloat: 1.1,
TopBool: true,
TopDate: tme,
TopArray: []string{"array"},
MatcH: "i should be in MatcH only",
Match: "i should be in Match only",
Once: "just once",
OncE: "",
Nest: InsensitiveNest{
Ed: InsensitiveEd{NestedString: "another string"},
},
}
var got Insensitive
if _, err := Decode(caseToml, &got); err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(expected, got) {
t.Fatalf("\n%#v\n!=\n%#v\n", expected, got)
}
}
func TestPointers(t *testing.T) {
type Object struct {
Type string
Description string
}
type Dict struct {
NamedObject map[string]*Object
BaseObject *Object
Strptr *string
Strptrs []*string
}
s1, s2, s3 := "blah", "abc", "def"
expected := &Dict{
Strptr: &s1,
Strptrs: []*string{&s2, &s3},
NamedObject: map[string]*Object{
"foo": {"FOO", "fooooo!!!"},
"bar": {"BAR", "ba-ba-ba-ba-barrrr!!!"},
},
BaseObject: &Object{"BASE", "da base"},
}
ex1 := `
Strptr = "blah"
Strptrs = ["abc", "def"]
[NamedObject.foo]
Type = "FOO"
Description = "fooooo!!!"
[NamedObject.bar]
Type = "BAR"
Description = "ba-ba-ba-ba-barrrr!!!"
[BaseObject]
Type = "BASE"
Description = "da base"
`
dict := new(Dict)
_, err := Decode(ex1, dict)
if err != nil {
t.Errorf("Decode error: %v", err)
}
if !reflect.DeepEqual(expected, dict) {
t.Fatalf("\n%#v\n!=\n%#v\n", expected, dict)
}
}
type sphere struct {
Center [3]float64
Radius float64
}
func TestDecodeSimpleArray(t *testing.T) {
var s1 sphere
if _, err := Decode(`center = [0.0, 1.5, 0.0]`, &s1); err != nil {
t.Fatal(err)
}
}
func TestDecodeArrayWrongSize(t *testing.T) {
var s1 sphere
if _, err := Decode(`center = [0.1, 2.3]`, &s1); err == nil {
t.Fatal("Expected array type mismatch error")
}
}
func TestDecodeLargeIntoSmallInt(t *testing.T) {
type table struct {
Value int8
}
var tab table
if _, err := Decode(`value = 500`, &tab); err == nil {
t.Fatal("Expected integer out-of-bounds error.")
}
}
func TestDecodeSizedInts(t *testing.T) {
type table struct {
U8 uint8
U16 uint16
U32 uint32
U64 uint64
U uint
I8 int8
I16 int16
I32 int32
I64 int64
I int
}
answer := table{1, 1, 1, 1, 1, -1, -1, -1, -1, -1}
toml := `
u8 = 1
u16 = 1
u32 = 1
u64 = 1
u = 1
i8 = -1
i16 = -1
i32 = -1
i64 = -1
i = -1
`
var tab table
if _, err := Decode(toml, &tab); err != nil {
t.Fatal(err.Error())
}
if answer != tab {
t.Fatalf("Expected %#v but got %#v", answer, tab)
}
}
func TestUnmarshaler(t *testing.T) {
var tomlBlob = `
[dishes.hamboogie]
name = "Hamboogie with fries"
price = 10.99
[[dishes.hamboogie.ingredients]]
name = "Bread Bun"
[[dishes.hamboogie.ingredients]]
name = "Lettuce"
[[dishes.hamboogie.ingredients]]
name = "Real Beef Patty"
[[dishes.hamboogie.ingredients]]
name = "Tomato"
[dishes.eggsalad]
name = "Egg Salad with rice"
price = 3.99
[[dishes.eggsalad.ingredients]]
name = "Egg"
[[dishes.eggsalad.ingredients]]
name = "Mayo"
[[dishes.eggsalad.ingredients]]
name = "Rice"
`
m := &menu{}
if _, err := Decode(tomlBlob, m); err != nil {
log.Fatal(err)
}
if len(m.Dishes) != 2 {
t.Log("two dishes should be loaded with UnmarshalTOML()")
t.Errorf("expected %d but got %d", 2, len(m.Dishes))
}
eggSalad := m.Dishes["eggsalad"]
if _, ok := interface{}(eggSalad).(dish); !ok {
t.Errorf("expected a dish")
}
if eggSalad.Name != "Egg Salad with rice" {
t.Errorf("expected the dish to be named 'Egg Salad with rice'")
}
if len(eggSalad.Ingredients) != 3 {
t.Log("dish should be loaded with UnmarshalTOML()")
t.Errorf("expected %d but got %d", 3, len(eggSalad.Ingredients))
}
found := false
for _, i := range eggSalad.Ingredients {
if i.Name == "Rice" {
found = true
break
}
}
if !found {
t.Error("Rice was not loaded in UnmarshalTOML()")
}
// test on a value - must be passed as *
o := menu{}
if _, err := Decode(tomlBlob, &o); err != nil {
log.Fatal(err)
}
}
type menu struct {
Dishes map[string]dish
}
func (m *menu) UnmarshalTOML(p interface{}) error {
m.Dishes = make(map[string]dish)
data, _ := p.(map[string]interface{})
dishes := data["dishes"].(map[string]interface{})
for n, v := range dishes {
if d, ok := v.(map[string]interface{}); ok {
nd := dish{}
nd.UnmarshalTOML(d)
m.Dishes[n] = nd
} else {
return fmt.Errorf("not a dish")
}
}
return nil
}
type dish struct {
Name string
Price float32
Ingredients []ingredient
}
func (d *dish) UnmarshalTOML(p interface{}) error {
data, _ := p.(map[string]interface{})
d.Name, _ = data["name"].(string)
d.Price, _ = data["price"].(float32)
ingredients, _ := data["ingredients"].([]map[string]interface{})
for _, e := range ingredients {
n, _ := interface{}(e).(map[string]interface{})
name, _ := n["name"].(string)
i := ingredient{name}
d.Ingredients = append(d.Ingredients, i)
}
return nil
}
type ingredient struct {
Name string
}
func ExampleMetaData_PrimitiveDecode() {
var md MetaData
var err error
var tomlBlob = `
ranking = ["Springsteen", "J Geils"]
[bands.Springsteen]
started = 1973
albums = ["Greetings", "WIESS", "Born to Run", "Darkness"]
[bands."J Geils"]
started = 1970
albums = ["The J. Geils Band", "Full House", "Blow Your Face Out"]
`
type band struct {
Started int
Albums []string
}
type classics struct {
Ranking []string
Bands map[string]Primitive
}
// Do the initial decode. Reflection is delayed on Primitive values.
var music classics
if md, err = Decode(tomlBlob, &music); err != nil {
log.Fatal(err)
}
// MetaData still includes information on Primitive values.
fmt.Printf("Is `bands.Springsteen` defined? %v\n",
md.IsDefined("bands", "Springsteen"))
// Decode primitive data into Go values.
for _, artist := range music.Ranking {
// A band is a primitive value, so we need to decode it to get a
// real `band` value.
primValue := music.Bands[artist]
var aBand band
if err = md.PrimitiveDecode(primValue, &aBand); err != nil {
log.Fatal(err)
}
fmt.Printf("%s started in %d.\n", artist, aBand.Started)
}
// Check to see if there were any fields left undecoded.
// Note that this won't be empty before decoding the Primitive value!
fmt.Printf("Undecoded: %q\n", md.Undecoded())
// Output:
// Is `bands.Springsteen` defined? true
// Springsteen started in 1973.
// J Geils started in 1970.
// Undecoded: []
}
func ExampleDecode() {
var tomlBlob = `
# Some comments.
[alpha]
ip = "10.0.0.1"
[alpha.config]
Ports = [ 8001, 8002 ]
Location = "Toronto"
Created = 1987-07-05T05:45:00Z
[beta]
ip = "10.0.0.2"
[beta.config]
Ports = [ 9001, 9002 ]
Location = "New Jersey"
Created = 1887-01-05T05:55:00Z
`
type serverConfig struct {
Ports []int
Location string
Created time.Time
}
type server struct {
IP string `toml:"ip"`
Config serverConfig `toml:"config"`
}
type servers map[string]server
var config servers
if _, err := Decode(tomlBlob, &config); err != nil {
log.Fatal(err)
}
for _, name := range []string{"alpha", "beta"} {
s := config[name]
fmt.Printf("Server: %s (ip: %s) in %s created on %s\n",
name, s.IP, s.Config.Location,
s.Config.Created.Format("2006-01-02"))
fmt.Printf("Ports: %v\n", s.Config.Ports)
}
// Output:
// Server: alpha (ip: 10.0.0.1) in Toronto created on 1987-07-05
// Ports: [8001 8002]
// Server: beta (ip: 10.0.0.2) in New Jersey created on 1887-01-05
// Ports: [9001 9002]
}
type duration struct {
time.Duration
}
func (d *duration) UnmarshalText(text []byte) error {
var err error
d.Duration, err = time.ParseDuration(string(text))
return err
}
// Example Unmarshaler shows how to decode TOML strings into your own
// custom data type.
func Example_unmarshaler() {
blob := `
[[song]]
name = "Thunder Road"
duration = "4m49s"
[[song]]
name = "Stairway to Heaven"
duration = "8m03s"
`
type song struct {
Name string
Duration duration
}
type songs struct {
Song []song
}
var favorites songs
if _, err := Decode(blob, &favorites); err != nil {
log.Fatal(err)
}
// Code to implement the TextUnmarshaler interface for `duration`:
//
// type duration struct {
// time.Duration
// }
//
// func (d *duration) UnmarshalText(text []byte) error {
// var err error
// d.Duration, err = time.ParseDuration(string(text))
// return err
// }
for _, s := range favorites.Song {
fmt.Printf("%s (%s)\n", s.Name, s.Duration)
}
// Output:
// Thunder Road (4m49s)
// Stairway to Heaven (8m3s)
}
// Example StrictDecoding shows how to detect whether there are keys in the
// TOML document that weren't decoded into the value given. This is useful
// for returning an error to the user if they've included extraneous fields
// in their configuration.
func Example_strictDecoding() {
var blob = `
key1 = "value1"
key2 = "value2"
key3 = "value3"
`
type config struct {
Key1 string
Key3 string
}
var conf config
md, err := Decode(blob, &conf)
if err != nil {
log.Fatal(err)
}
fmt.Printf("Undecoded keys: %q\n", md.Undecoded())
// Output:
// Undecoded keys: ["key2"]
}
// Example UnmarshalTOML shows how to implement a struct type that knows how to
// unmarshal itself. The struct must take full responsibility for mapping the
// values passed into the struct. The method may be used with interfaces in a
// struct in cases where the actual type is not known until the data is
// examined.
func Example_unmarshalTOML() {
var blob = `
[[parts]]
type = "valve"
id = "valve-1"
size = 1.2
rating = 4
[[parts]]
type = "valve"
id = "valve-2"
size = 2.1
rating = 5
[[parts]]
type = "pipe"
id = "pipe-1"
length = 2.1
diameter = 12
[[parts]]
type = "cable"
id = "cable-1"
length = 12
rating = 3.1
`
o := &order{}
err := Unmarshal([]byte(blob), o)
if err != nil {
log.Fatal(err)
}
fmt.Println(len(o.parts))
for _, part := range o.parts {
fmt.Println(part.Name())
}
// Code to implement UmarshalJSON.
// type order struct {
// // NOTE `order.parts` is a private slice of type `part` which is an
// // interface and may only be loaded from toml using the
// // UnmarshalTOML() method of the Umarshaler interface.
// parts parts
// }
// func (o *order) UnmarshalTOML(data interface{}) error {
// // NOTE the example below contains detailed type casting to show how
// // the 'data' is retrieved. In operational use, a type cast wrapper
// // may be prefered e.g.
// //
// // func AsMap(v interface{}) (map[string]interface{}, error) {
// // return v.(map[string]interface{})
// // }
// //
// // resulting in:
// // d, _ := AsMap(data)
// //
// d, _ := data.(map[string]interface{})
// parts, _ := d["parts"].([]map[string]interface{})
// for _, p := range parts {
// typ, _ := p["type"].(string)
// id, _ := p["id"].(string)
// // detect the type of part and handle each case
// switch p["type"] {
// case "valve":
// size := float32(p["size"].(float64))
// rating := int(p["rating"].(int64))
// valve := &valve{
// Type: typ,
// ID: id,
// Size: size,
// Rating: rating,
// }
// o.parts = append(o.parts, valve)
// case "pipe":
// length := float32(p["length"].(float64))
// diameter := int(p["diameter"].(int64))
// pipe := &pipe{
// Type: typ,
// ID: id,
// Length: length,
// Diameter: diameter,
// }
// o.parts = append(o.parts, pipe)
// case "cable":
// length := int(p["length"].(int64))
// rating := float32(p["rating"].(float64))
// cable := &cable{
// Type: typ,
// ID: id,
// Length: length,
// Rating: rating,
// }
// o.parts = append(o.parts, cable)
// }
// }
// return nil
// }
// type parts []part
// type part interface {
// Name() string
// }
// type valve struct {
// Type string
// ID string
// Size float32
// Rating int
// }
// func (v *valve) Name() string {
// return fmt.Sprintf("VALVE: %s", v.ID)
// }
// type pipe struct {
// Type string
// ID string
// Length float32
// Diameter int
// }
// func (p *pipe) Name() string {
// return fmt.Sprintf("PIPE: %s", p.ID)
// }
// type cable struct {
// Type string
// ID string
// Length int
// Rating float32
// }
// func (c *cable) Name() string {
// return fmt.Sprintf("CABLE: %s", c.ID)
// }
// Output:
// 4
// VALVE: valve-1
// VALVE: valve-2
// PIPE: pipe-1
// CABLE: cable-1
}
type order struct {
// NOTE `order.parts` is a private slice of type `part` which is an
// interface and may only be loaded from toml using the UnmarshalTOML()
// method of the Umarshaler interface.
parts parts
}
func (o *order) UnmarshalTOML(data interface{}) error {
// NOTE the example below contains detailed type casting to show how
// the 'data' is retrieved. In operational use, a type cast wrapper
// may be prefered e.g.
//
// func AsMap(v interface{}) (map[string]interface{}, error) {
// return v.(map[string]interface{})
// }
//
// resulting in:
// d, _ := AsMap(data)
//
d, _ := data.(map[string]interface{})
parts, _ := d["parts"].([]map[string]interface{})
for _, p := range parts {
typ, _ := p["type"].(string)
id, _ := p["id"].(string)
// detect the type of part and handle each case
switch p["type"] {
case "valve":
size := float32(p["size"].(float64))
rating := int(p["rating"].(int64))
valve := &valve{
Type: typ,
ID: id,
Size: size,
Rating: rating,
}
o.parts = append(o.parts, valve)
case "pipe":
length := float32(p["length"].(float64))
diameter := int(p["diameter"].(int64))
pipe := &pipe{
Type: typ,
ID: id,
Length: length,
Diameter: diameter,
}
o.parts = append(o.parts, pipe)
case "cable":
length := int(p["length"].(int64))
rating := float32(p["rating"].(float64))
cable := &cable{
Type: typ,
ID: id,
Length: length,
Rating: rating,
}
o.parts = append(o.parts, cable)
}
}
return nil
}
type parts []part
type part interface {
Name() string
}
type valve struct {
Type string
ID string
Size float32
Rating int
}
func (v *valve) Name() string {
return fmt.Sprintf("VALVE: %s", v.ID)
}
type pipe struct {
Type string
ID string
Length float32
Diameter int
}
func (p *pipe) Name() string {
return fmt.Sprintf("PIPE: %s", p.ID)
}
type cable struct {
Type string
ID string
Length int
Rating float32
}
func (c *cable) Name() string {
return fmt.Sprintf("CABLE: %s", c.ID)
}

542
vendor/github.com/BurntSushi/toml/encode_test.go generated vendored
View file

@ -1,542 +0,0 @@
package toml
import (
"bytes"
"fmt"
"log"
"net"
"testing"
"time"
)
func TestEncodeRoundTrip(t *testing.T) {
type Config struct {
Age int
Cats []string
Pi float64
Perfection []int
DOB time.Time
Ipaddress net.IP
}
var inputs = Config{
13,
[]string{"one", "two", "three"},
3.145,
[]int{11, 2, 3, 4},
time.Now(),
net.ParseIP("192.168.59.254"),
}
var firstBuffer bytes.Buffer
e := NewEncoder(&firstBuffer)
err := e.Encode(inputs)
if err != nil {
t.Fatal(err)
}
var outputs Config
if _, err := Decode(firstBuffer.String(), &outputs); err != nil {
log.Printf("Could not decode:\n-----\n%s\n-----\n",
firstBuffer.String())
t.Fatal(err)
}
// could test each value individually, but I'm lazy
var secondBuffer bytes.Buffer
e2 := NewEncoder(&secondBuffer)
err = e2.Encode(outputs)
if err != nil {
t.Fatal(err)
}
if firstBuffer.String() != secondBuffer.String() {
t.Error(
firstBuffer.String(),
"\n\n is not identical to\n\n",
secondBuffer.String())
}
}
// XXX(burntsushi)
// I think these tests probably should be removed. They are good, but they
// ought to be obsolete by toml-test.
func TestEncode(t *testing.T) {
type Embedded struct {
Int int `toml:"_int"`
}
type NonStruct int
date := time.Date(2014, 5, 11, 20, 30, 40, 0, time.FixedZone("IST", 3600))
dateStr := "2014-05-11T19:30:40Z"
tests := map[string]struct {
input interface{}
wantOutput string
wantError error
}{
"bool field": {
input: struct {
BoolTrue bool
BoolFalse bool
}{true, false},
wantOutput: "BoolTrue = true\nBoolFalse = false\n",
},
"int fields": {
input: struct {
Int int
Int8 int8
Int16 int16
Int32 int32
Int64 int64
}{1, 2, 3, 4, 5},
wantOutput: "Int = 1\nInt8 = 2\nInt16 = 3\nInt32 = 4\nInt64 = 5\n",
},
"uint fields": {
input: struct {
Uint uint
Uint8 uint8
Uint16 uint16
Uint32 uint32
Uint64 uint64
}{1, 2, 3, 4, 5},
wantOutput: "Uint = 1\nUint8 = 2\nUint16 = 3\nUint32 = 4" +
"\nUint64 = 5\n",
},
"float fields": {
input: struct {
Float32 float32
Float64 float64
}{1.5, 2.5},
wantOutput: "Float32 = 1.5\nFloat64 = 2.5\n",
},
"string field": {
input: struct{ String string }{"foo"},
wantOutput: "String = \"foo\"\n",
},
"string field and unexported field": {
input: struct {
String string
unexported int
}{"foo", 0},
wantOutput: "String = \"foo\"\n",
},
"datetime field in UTC": {
input: struct{ Date time.Time }{date},
wantOutput: fmt.Sprintf("Date = %s\n", dateStr),
},
"datetime field as primitive": {
// Using a map here to fail if isStructOrMap() returns true for
// time.Time.
input: map[string]interface{}{
"Date": date,
"Int": 1,
},
wantOutput: fmt.Sprintf("Date = %s\nInt = 1\n", dateStr),
},
"array fields": {
input: struct {
IntArray0 [0]int
IntArray3 [3]int
}{[0]int{}, [3]int{1, 2, 3}},
wantOutput: "IntArray0 = []\nIntArray3 = [1, 2, 3]\n",
},
"slice fields": {
input: struct{ IntSliceNil, IntSlice0, IntSlice3 []int }{
nil, []int{}, []int{1, 2, 3},
},
wantOutput: "IntSlice0 = []\nIntSlice3 = [1, 2, 3]\n",
},
"datetime slices": {
input: struct{ DatetimeSlice []time.Time }{
[]time.Time{date, date},
},
wantOutput: fmt.Sprintf("DatetimeSlice = [%s, %s]\n",
dateStr, dateStr),
},
"nested arrays and slices": {
input: struct {
SliceOfArrays [][2]int
ArrayOfSlices [2][]int
SliceOfArraysOfSlices [][2][]int
ArrayOfSlicesOfArrays [2][][2]int
SliceOfMixedArrays [][2]interface{}
ArrayOfMixedSlices [2][]interface{}
}{
[][2]int{{1, 2}, {3, 4}},
[2][]int{{1, 2}, {3, 4}},
[][2][]int{
{
{1, 2}, {3, 4},
},
{
{5, 6}, {7, 8},
},
},
[2][][2]int{
{
{1, 2}, {3, 4},
},
{
{5, 6}, {7, 8},
},
},
[][2]interface{}{
{1, 2}, {"a", "b"},
},
[2][]interface{}{
{1, 2}, {"a", "b"},
},
},
wantOutput: `SliceOfArrays = [[1, 2], [3, 4]]
ArrayOfSlices = [[1, 2], [3, 4]]
SliceOfArraysOfSlices = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
ArrayOfSlicesOfArrays = [[[1, 2], [3, 4]], [[5, 6], [7, 8]]]
SliceOfMixedArrays = [[1, 2], ["a", "b"]]
ArrayOfMixedSlices = [[1, 2], ["a", "b"]]
`,
},
"empty slice": {
input: struct{ Empty []interface{} }{[]interface{}{}},
wantOutput: "Empty = []\n",
},
"(error) slice with element type mismatch (string and integer)": {
input: struct{ Mixed []interface{} }{[]interface{}{1, "a"}},
wantError: errArrayMixedElementTypes,
},
"(error) slice with element type mismatch (integer and float)": {
input: struct{ Mixed []interface{} }{[]interface{}{1, 2.5}},
wantError: errArrayMixedElementTypes,
},
"slice with elems of differing Go types, same TOML types": {
input: struct {
MixedInts []interface{}
MixedFloats []interface{}
}{
[]interface{}{
int(1), int8(2), int16(3), int32(4), int64(5),
uint(1), uint8(2), uint16(3), uint32(4), uint64(5),
},
[]interface{}{float32(1.5), float64(2.5)},
},
wantOutput: "MixedInts = [1, 2, 3, 4, 5, 1, 2, 3, 4, 5]\n" +
"MixedFloats = [1.5, 2.5]\n",
},
"(error) slice w/ element type mismatch (one is nested array)": {
input: struct{ Mixed []interface{} }{
[]interface{}{1, []interface{}{2}},
},
wantError: errArrayMixedElementTypes,
},
"(error) slice with 1 nil element": {
input: struct{ NilElement1 []interface{} }{[]interface{}{nil}},
wantError: errArrayNilElement,
},
"(error) slice with 1 nil element (and other non-nil elements)": {
input: struct{ NilElement []interface{} }{
[]interface{}{1, nil},
},
wantError: errArrayNilElement,
},
"simple map": {
input: map[string]int{"a": 1, "b": 2},
wantOutput: "a = 1\nb = 2\n",
},
"map with interface{} value type": {
input: map[string]interface{}{"a": 1, "b": "c"},
wantOutput: "a = 1\nb = \"c\"\n",
},
"map with interface{} value type, some of which are structs": {
input: map[string]interface{}{
"a": struct{ Int int }{2},
"b": 1,
},
wantOutput: "b = 1\n\n[a]\n Int = 2\n",
},
"nested map": {
input: map[string]map[string]int{
"a": {"b": 1},
"c": {"d": 2},
},
wantOutput: "[a]\n b = 1\n\n[c]\n d = 2\n",
},
"nested struct": {
input: struct{ Struct struct{ Int int } }{
struct{ Int int }{1},
},
wantOutput: "[Struct]\n Int = 1\n",
},
"nested struct and non-struct field": {
input: struct {
Struct struct{ Int int }
Bool bool
}{struct{ Int int }{1}, true},
wantOutput: "Bool = true\n\n[Struct]\n Int = 1\n",
},
"2 nested structs": {
input: struct{ Struct1, Struct2 struct{ Int int } }{
struct{ Int int }{1}, struct{ Int int }{2},
},
wantOutput: "[Struct1]\n Int = 1\n\n[Struct2]\n Int = 2\n",
},
"deeply nested structs": {
input: struct {
Struct1, Struct2 struct{ Struct3 *struct{ Int int } }
}{
struct{ Struct3 *struct{ Int int } }{&struct{ Int int }{1}},
struct{ Struct3 *struct{ Int int } }{nil},
},
wantOutput: "[Struct1]\n [Struct1.Struct3]\n Int = 1" +
"\n\n[Struct2]\n",
},
"nested struct with nil struct elem": {
input: struct {
Struct struct{ Inner *struct{ Int int } }
}{
struct{ Inner *struct{ Int int } }{nil},
},
wantOutput: "[Struct]\n",
},
"nested struct with no fields": {
input: struct {
Struct struct{ Inner struct{} }
}{
struct{ Inner struct{} }{struct{}{}},
},
wantOutput: "[Struct]\n [Struct.Inner]\n",
},
"struct with tags": {
input: struct {
Struct struct {
Int int `toml:"_int"`
} `toml:"_struct"`
Bool bool `toml:"_bool"`
}{
struct {
Int int `toml:"_int"`
}{1}, true,
},
wantOutput: "_bool = true\n\n[_struct]\n _int = 1\n",
},
"embedded struct": {
input: struct{ Embedded }{Embedded{1}},
wantOutput: "_int = 1\n",
},
"embedded *struct": {
input: struct{ *Embedded }{&Embedded{1}},
wantOutput: "_int = 1\n",
},
"nested embedded struct": {
input: struct {
Struct struct{ Embedded } `toml:"_struct"`
}{struct{ Embedded }{Embedded{1}}},
wantOutput: "[_struct]\n _int = 1\n",
},
"nested embedded *struct": {
input: struct {
Struct struct{ *Embedded } `toml:"_struct"`
}{struct{ *Embedded }{&Embedded{1}}},
wantOutput: "[_struct]\n _int = 1\n",
},
"array of tables": {
input: struct {
Structs []*struct{ Int int } `toml:"struct"`
}{
[]*struct{ Int int }{{1}, {3}},
},
wantOutput: "[[struct]]\n Int = 1\n\n[[struct]]\n Int = 3\n",
},
"array of tables order": {
input: map[string]interface{}{
"map": map[string]interface{}{
"zero": 5,
"arr": []map[string]int{
map[string]int{
"friend": 5,
},
},
},
},
wantOutput: "[map]\n zero = 5\n\n [[map.arr]]\n friend = 5\n",
},
"(error) top-level slice": {
input: []struct{ Int int }{{1}, {2}, {3}},
wantError: errNoKey,
},
"(error) slice of slice": {
input: struct {
Slices [][]struct{ Int int }
}{
[][]struct{ Int int }{{{1}}, {{2}}, {{3}}},
},
wantError: errArrayNoTable,
},
"(error) map no string key": {
input: map[int]string{1: ""},
wantError: errNonString,
},
"(error) anonymous non-struct": {
input: struct{ NonStruct }{5},
wantError: errAnonNonStruct,
},
"(error) empty key name": {
input: map[string]int{"": 1},
wantError: errAnything,
},
"(error) empty map name": {
input: map[string]interface{}{
"": map[string]int{"v": 1},
},
wantError: errAnything,
},
}
for label, test := range tests {
encodeExpected(t, label, test.input, test.wantOutput, test.wantError)
}
}
func TestEncodeNestedTableArrays(t *testing.T) {
type song struct {
Name string `toml:"name"`
}
type album struct {
Name string `toml:"name"`
Songs []song `toml:"songs"`
}
type springsteen struct {
Albums []album `toml:"albums"`
}
value := springsteen{
[]album{
{"Born to Run",
[]song{{"Jungleland"}, {"Meeting Across the River"}}},
{"Born in the USA",
[]song{{"Glory Days"}, {"Dancing in the Dark"}}},
},
}
expected := `[[albums]]
name = "Born to Run"
[[albums.songs]]
name = "Jungleland"
[[albums.songs]]
name = "Meeting Across the River"
[[albums]]
name = "Born in the USA"
[[albums.songs]]
name = "Glory Days"
[[albums.songs]]
name = "Dancing in the Dark"
`
encodeExpected(t, "nested table arrays", value, expected, nil)
}
func TestEncodeArrayHashWithNormalHashOrder(t *testing.T) {
type Alpha struct {
V int
}
type Beta struct {
V int
}
type Conf struct {
V int
A Alpha
B []Beta
}
val := Conf{
V: 1,
A: Alpha{2},
B: []Beta{{3}},
}
expected := "V = 1\n\n[A]\n V = 2\n\n[[B]]\n V = 3\n"
encodeExpected(t, "array hash with normal hash order", val, expected, nil)
}
func TestEncodeWithOmitEmpty(t *testing.T) {
type simple struct {
User string `toml:"user"`
Pass string `toml:"password,omitempty"`
}
value := simple{"Testing", ""}
expected := fmt.Sprintf("user = %q\n", value.User)
encodeExpected(t, "simple with omitempty, is empty", value, expected, nil)
value.Pass = "some password"
expected = fmt.Sprintf("user = %q\npassword = %q\n", value.User, value.Pass)
encodeExpected(t, "simple with omitempty, not empty", value, expected, nil)
}
func TestEncodeWithOmitZero(t *testing.T) {
type simple struct {
Number int `toml:"number,omitzero"`
Real float64 `toml:"real,omitzero"`
Unsigned uint `toml:"unsigned,omitzero"`
}
value := simple{0, 0.0, uint(0)}
expected := ""
encodeExpected(t, "simple with omitzero, all zero", value, expected, nil)
value.Number = 10
value.Real = 20
value.Unsigned = 5
expected = `number = 10
real = 20.0
unsigned = 5
`
encodeExpected(t, "simple with omitzero, non-zero", value, expected, nil)
}
func encodeExpected(
t *testing.T, label string, val interface{}, wantStr string, wantErr error,
) {
var buf bytes.Buffer
enc := NewEncoder(&buf)
err := enc.Encode(val)
if err != wantErr {
if wantErr != nil {
if wantErr == errAnything && err != nil {
return
}
t.Errorf("%s: want Encode error %v, got %v", label, wantErr, err)
} else {
t.Errorf("%s: Encode failed: %s", label, err)
}
}
if err != nil {
return
}
if got := buf.String(); wantStr != got {
t.Errorf("%s: want\n-----\n%q\n-----\nbut got\n-----\n%q\n-----\n",
label, wantStr, got)
}
}
func ExampleEncoder_Encode() {
date, _ := time.Parse(time.RFC822, "14 Mar 10 18:00 UTC")
var config = map[string]interface{}{
"date": date,
"counts": []int{1, 1, 2, 3, 5, 8},
"hash": map[string]string{
"key1": "val1",
"key2": "val2",
},
}
buf := new(bytes.Buffer)
if err := NewEncoder(buf).Encode(config); err != nil {
log.Fatal(err)
}
fmt.Println(buf.String())
// Output:
// counts = [1, 1, 2, 3, 5, 8]
// date = 2010-03-14T18:00:00Z
//
// [hash]
// key1 = "val1"
// key2 = "val2"
}

22
vendor/github.com/bitly/go-notify/.gitignore generated vendored
View file

@ -1,22 +0,0 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe

51
vendor/github.com/bitly/go-notify/README.md generated vendored
View file

@ -1,51 +0,0 @@
## go-notify
Package notify enables independent components of an application to
observe notable events in a decoupled fashion.
It generalizes the pattern of *multiple* consumers of an event (ie: the
same message delivered to multiple channels) and obviates the need for
components to have intimate knowledge of each other (only `import
notify` and the name of the event are shared).
Example:
// producer of "my_event"
go func() {
for {
time.Sleep(time.Duration(1) * time.Second):
notify.Post("my_event", time.Now().Unix())
}
}()
// observer of "my_event" (normally some independent component that
// needs to be notified when "my_event" occurs)
myEventChan := make(chan interface{})
notify.Start("my_event", myEventChan)
go func() {
for {
data := <-myEventChan
log.Printf("MY_EVENT: %#v", data)
}
}()
### Functions
func Post(event string, data interface{}) error
Post a notification (arbitrary data) to the specified event
func PostTimeout(event string, data interface{}, timeout time.Duration) error
Post a notification to the specified event using the provided timeout for
any output channels that are blocking
func Start(event string, outputChan chan interface{})
Start observing the specified event via provided output channel
func Stop(event string, outputChan chan interface{}) error
Stop observing the specified event on the provided output channel
func StopAll(event string) error
Stop observing the specified event on all channels
func Version() string
returns the current version

130
vendor/github.com/bitly/go-notify/notify.go generated vendored
View file

@ -1,130 +0,0 @@
// Package notify enables independent components of an application to
// observe notable events in a decoupled fashion.
//
// It generalizes the pattern of *multiple* consumers of an event (ie:
// the same message delivered to multiple channels) and obviates the need
// for components to have intimate knowledge of each other (only `import notify`
// and the name of the event are shared).
//
// Example:
// // producer of "my_event"
// go func() {
// for {
// time.Sleep(time.Duration(1) * time.Second):
// notify.Post("my_event", time.Now().Unix())
// }
// }()
//
// // observer of "my_event" (normally some independent component that
// // needs to be notified when "my_event" occurs)
// myEventChan := make(chan interface{})
// notify.Start("my_event", myEventChan)
// go func() {
// for {
// data := <-myEventChan
// log.Printf("MY_EVENT: %#v", data)
// }
// }()
package notify
import (
"errors"
"sync"
"time"
)
const E_NOT_FOUND = "E_NOT_FOUND"
// returns the current version
func Version() string {
return "0.2"
}
// internal mapping of event names to observing channels
var events = make(map[string][]chan interface{})
// mutex for touching the event map
var rwMutex sync.RWMutex
// Start observing the specified event via provided output channel
func Start(event string, outputChan chan interface{}) {
rwMutex.Lock()
defer rwMutex.Unlock()
events[event] = append(events[event], outputChan)
}
// Stop observing the specified event on the provided output channel
func Stop(event string, outputChan chan interface{}) error {
rwMutex.Lock()
defer rwMutex.Unlock()
newArray := make([]chan interface{}, 0)
outChans, ok := events[event]
if !ok {
return errors.New(E_NOT_FOUND)
}
for _, ch := range outChans {
if ch != outputChan {
newArray = append(newArray, ch)
} else {
close(ch)
}
}
events[event] = newArray
return nil
}
// Stop observing the specified event on all channels
func StopAll(event string) error {
rwMutex.Lock()
defer rwMutex.Unlock()
outChans, ok := events[event]
if !ok {
return errors.New(E_NOT_FOUND)
}
for _, ch := range outChans {
close(ch)
}
delete(events, event)
return nil
}
// Post a notification (arbitrary data) to the specified event
func Post(event string, data interface{}) error {
rwMutex.RLock()
defer rwMutex.RUnlock()
outChans, ok := events[event]
if !ok {
return errors.New(E_NOT_FOUND)
}
for _, outputChan := range outChans {
outputChan <- data
}
return nil
}
// Post a notification to the specified event using the provided timeout for
// any output channels that are blocking
func PostTimeout(event string, data interface{}, timeout time.Duration) error {
rwMutex.RLock()
defer rwMutex.RUnlock()
outChans, ok := events[event]
if !ok {
return errors.New(E_NOT_FOUND)
}
for _, outputChan := range outChans {
select {
case outputChan <- data:
case <-time.After(timeout):
}
}
return nil
}

25
vendor/github.com/cactus/go-statsd-client/statsd/doc.go generated vendored
View file

@ -1,25 +0,0 @@
/*
Package statsd provides a StatsD client implementation that is safe for
concurrent use by multiple goroutines and for efficiency can be created and
reused.
Example usage:
// first create a client
client, err := statsd.New("127.0.0.1:8125", "test-client")
// handle any errors
if err != nil {
log.Fatal(err)
}
// make sure to clean up
defer client.Close()
// Send a stat
err = client.Inc("stat1", 42, 1.0)
// handle any errors
if err != nil {
log.Printf("Error sending metric: %+v", err)
}
*/
package statsd

151
vendor/github.com/cactus/go-statsd-client/statsd/main.go generated vendored
View file

@ -1,151 +0,0 @@
package statsd
import (
"errors"
"fmt"
"math/rand"
"net"
)
type Statter interface {
Inc(stat string, value int64, rate float32) error
Dec(stat string, value int64, rate float32) error
Gauge(stat string, value int64, rate float32) error
GaugeDelta(stat string, value int64, rate float32) error
Timing(stat string, delta int64, rate float32) error
Raw(stat string, value string, rate float32) error
SetPrefix(prefix string)
Close() error
}
type Client struct {
// underlying connection
c net.PacketConn
// resolved udp address
ra *net.UDPAddr
// prefix for statsd name
prefix string
}
// Close closes the connection and cleans up.
func (s *Client) Close() error {
err := s.c.Close()
return err
}
// Increments a statsd count type.
// stat is a string name for the metric.
// value is the integer value
// rate is the sample rate (0.0 to 1.0)
func (s *Client) Inc(stat string, value int64, rate float32) error {
dap := fmt.Sprintf("%d|c", value)
return s.Raw(stat, dap, rate)
}
// Decrements a statsd count type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Dec(stat string, value int64, rate float32) error {
return s.Inc(stat, -value, rate)
}
// Submits/Updates a statsd gauge type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Gauge(stat string, value int64, rate float32) error {
dap := fmt.Sprintf("%d|g", value)
return s.Raw(stat, dap, rate)
}
// Submits a delta to a statsd gauge.
// stat is the string name for the metric.
// value is the (positive or negative) change.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) GaugeDelta(stat string, value int64, rate float32) error {
dap := fmt.Sprintf("%+d|g", value)
return s.Raw(stat, dap, rate)
}
// Submits a statsd timing type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Timing(stat string, delta int64, rate float32) error {
dap := fmt.Sprintf("%d|ms", delta)
return s.Raw(stat, dap, rate)
}
// Raw formats the statsd event data, handles sampling, prepares it,
// and sends it to the server.
// stat is the string name for the metric.
// value is a preformatted "raw" value string.
// rate is the sample rate (0.0 to 1.0).
func (s *Client) Raw(stat string, value string, rate float32) error {
if rate < 1 {
if rand.Float32() < rate {
value = fmt.Sprintf("%s|@%f", value, rate)
} else {
return nil
}
}
if s.prefix != "" {
stat = fmt.Sprintf("%s.%s", s.prefix, stat)
}
data := fmt.Sprintf("%s:%s", stat, value)
_, err := s.send([]byte(data))
if err != nil {
return err
}
return nil
}
// Sets/Updates the statsd client prefix
func (s *Client) SetPrefix(prefix string) {
s.prefix = prefix
}
// sends the data to the server endpoint
func (s *Client) send(data []byte) (int, error) {
// no need for locking here, as the underlying fdNet
// already serialized writes
n, err := s.c.(*net.UDPConn).WriteToUDP([]byte(data), s.ra)
if err != nil {
return 0, err
}
if n == 0 {
return n, errors.New("Wrote no bytes")
}
return n, nil
}
// Returns a pointer to a new Client, and an error.
// addr is a string of the format "hostname:port", and must be parsable by
// net.ResolveUDPAddr.
// prefix is the statsd client prefix. Can be "" if no prefix is desired.
func New(addr, prefix string) (*Client, error) {
c, err := net.ListenPacket("udp", ":0")
if err != nil {
return nil, err
}
ra, err := net.ResolveUDPAddr("udp", addr)
if err != nil {
return nil, err
}
client := &Client{
c: c,
ra: ra,
prefix: prefix}
return client, nil
}
// Compatibility alias
var Dial = New

152
vendor/github.com/cactus/go-statsd-client/statsd/main_test.go generated vendored
View file

@ -1,152 +0,0 @@
package statsd
import (
"bytes"
"log"
"net"
"reflect"
"testing"
"time"
)
var statsdPacketTests = []struct {
Prefix string
Method string
Stat string
Value int64
Rate float32
Expected string
}{
{"test", "Gauge", "gauge", 1, 1.0, "test.gauge:1|g"},
{"test", "Inc", "count", 1, 0.999999, "test.count:1|c|@0.999999"},
{"test", "Inc", "count", 1, 1.0, "test.count:1|c"},
{"test", "Dec", "count", 1, 1.0, "test.count:-1|c"},
{"test", "Timing", "timing", 1, 1.0, "test.timing:1|ms"},
{"", "Inc", "count", 1, 1.0, "count:1|c"},
{"", "GaugeDelta", "gauge", 1, 1.0, "gauge:+1|g"},
{"", "GaugeDelta", "gauge", -1, 1.0, "gauge:-1|g"},
}
func TestClient(t *testing.T) {
l, err := newUDPListener("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
for _, tt := range statsdPacketTests {
c, err := New(l.LocalAddr().String(), tt.Prefix)
if err != nil {
t.Fatal(err)
}
method := reflect.ValueOf(c).MethodByName(tt.Method)
e := method.Call([]reflect.Value{
reflect.ValueOf(tt.Stat),
reflect.ValueOf(tt.Value),
reflect.ValueOf(tt.Rate)})[0]
errInter := e.Interface()
if errInter != nil {
t.Fatal(errInter.(error))
}
data := make([]byte, 128)
_, _, err = l.ReadFrom(data)
if err != nil {
c.Close()
t.Fatal(err)
}
data = bytes.TrimRight(data, "\x00")
if bytes.Equal(data, []byte(tt.Expected)) != true {
c.Close()
t.Fatalf("%s got '%s' expected '%s'", tt.Method, data, tt.Expected)
}
c.Close()
}
}
func TestNoopClient(t *testing.T) {
l, err := newUDPListener("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer l.Close()
for _, tt := range statsdPacketTests {
c, err := NewNoop(l.LocalAddr().String(), tt.Prefix)
if err != nil {
t.Fatal(err)
}
method := reflect.ValueOf(c).MethodByName(tt.Method)
e := method.Call([]reflect.Value{
reflect.ValueOf(tt.Stat),
reflect.ValueOf(tt.Value),
reflect.ValueOf(tt.Rate)})[0]
errInter := e.Interface()
if errInter != nil {
t.Fatal(errInter.(error))
}
data := make([]byte, 128)
n, _, err := l.ReadFrom(data)
// this is expected to error, since there should
// be no udp data sent, so the read will time out
if err == nil || n != 0 {
c.Close()
t.Fatal(err)
}
c.Close()
}
}
func newUDPListener(addr string) (*net.UDPConn, error) {
l, err := net.ListenPacket("udp", addr)
if err != nil {
return nil, err
}
l.SetDeadline(time.Now().Add(100 * time.Millisecond))
l.SetReadDeadline(time.Now().Add(100 * time.Millisecond))
l.SetWriteDeadline(time.Now().Add(100 * time.Millisecond))
return l.(*net.UDPConn), nil
}
func ExampleClient() {
// first create a client
client, err := Dial("127.0.0.1:8125", "test-client")
// handle any errors
if err != nil {
log.Fatal(err)
}
// make sure to clean up
defer client.Close()
// Send a stat
err = client.Inc("stat1", 42, 1.0)
// handle any errors
if err != nil {
log.Printf("Error sending metric: %+v", err)
}
}
func ExampleNoopClient() {
// use interface so we can sub noop client if needed
var client Statter
var err error
// first try to create a real client
client, err = Dial("not-resolvable:8125", "test-client")
// Lets say real client creation fails, but you don't care enough about
// stats that you don't want your program to run. Just log an error and
// make a NoopClient instead
if err != nil {
log.Println("Remote endpoint did not resolve. Disabling stats", err)
client, err = NewNoop()
}
// make sure to clean up
defer client.Close()
// Send a stat
err = client.Inc("stat1", 42, 1.0)
// handle any errors
if err != nil {
log.Printf("Error sending metric: %+v", err)
}
}

74
vendor/github.com/cactus/go-statsd-client/statsd/noop.go generated vendored
View file

@ -1,74 +0,0 @@
package statsd
type NoopClient struct {
// prefix for statsd name
prefix string
}
// Close closes the connection and cleans up.
func (s *NoopClient) Close() error {
return nil
}
// Increments a statsd count type.
// stat is a string name for the metric.
// value is the integer value
// rate is the sample rate (0.0 to 1.0)
func (s *NoopClient) Inc(stat string, value int64, rate float32) error {
return nil
}
// Decrements a statsd count type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *NoopClient) Dec(stat string, value int64, rate float32) error {
return nil
}
// Submits/Updates a statsd gauge type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *NoopClient) Gauge(stat string, value int64, rate float32) error {
return nil
}
// Submits a delta to a statsd gauge.
// stat is the string name for the metric.
// value is the (positive or negative) change.
// rate is the sample rate (0.0 to 1.0).
func (s *NoopClient) GaugeDelta(stat string, value int64, rate float32) error {
return nil
}
// Submits a statsd timing type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *NoopClient) Timing(stat string, delta int64, rate float32) error {
return nil
}
// Raw formats the statsd event data, handles sampling, prepares it,
// and sends it to the server.
// stat is the string name for the metric.
// value is the preformatted "raw" value string.
// rate is the sample rate (0.0 to 1.0).
func (s *NoopClient) Raw(stat string, value string, rate float32) error {
return nil
}
// Sets/Updates the statsd client prefix
func (s *NoopClient) SetPrefix(prefix string) {
s.prefix = prefix
}
// Returns a pointer to a new NoopClient, and an error (always nil, just
// supplied to support api convention).
// Use variadic arguments to support identical format as New, or a more
// conventional no argument form.
func NewNoop(a ...interface{}) (*NoopClient, error) {
noopClient := &NoopClient{}
return noopClient, nil
}

90
vendor/github.com/cactus/go-statsd-client/statsd/test-client/main.go generated vendored
View file

@ -1,90 +0,0 @@
package main
import (
"github.com/cactus/go-statsd-client/statsd"
flags "github.com/jessevdk/go-flags"
"fmt"
"log"
"os"
"time"
)
func main() {
// command line flags
var opts struct {
HostPort string `long:"host" default:"127.0.0.1:8125" description:"host:port of statsd server"`
Prefix string `long:"prefix" default:"test-client" description:"Statsd prefix"`
StatType string `long:"type" default:"count" description:"stat type to send. Can be timing, count, guage"`
StatValue int64 `long:"value" default:"1" description:"Value to send"`
Name string `short:"n" long:"name" default:"counter" description:"stat name"`
Rate float32 `short:"r" long:"rate" default:"1.0" description:"sample rate"`
Volume int `short:"c" long:"count" default:"1000" description:"Number of stats to send. Volume."`
Noop bool `long:"noop" default:"false" description:"Use noop client"`
Duration time.Duration `short:"d" long:"duration" default:"10s" description:"How long to spread the volume across. Each second of duration volume/seconds events will be sent."`
}
// parse said flags
_, err := flags.Parse(&opts)
if err != nil {
if e, ok := err.(*flags.Error); ok {
if e.Type == flags.ErrHelp {
os.Exit(0)
}
}
fmt.Printf("Error: %+v\n", err)
os.Exit(1)
}
var client statsd.Statter
if !opts.Noop {
client, err = statsd.New(opts.HostPort, opts.Prefix)
if err != nil {
log.Fatal(err)
}
defer client.Close()
} else {
client, err = statsd.NewNoop(opts.HostPort, opts.Prefix)
}
var stat func(stat string, value int64, rate float32) error
switch opts.StatType {
case "count":
stat = func(stat string, value int64, rate float32) error {
return client.Inc(stat, value, rate)
}
case "gauge":
stat = func(stat string, value int64, rate float32) error {
return client.Gauge(stat, value, rate)
}
case "timing":
stat = func(stat string, value int64, rate float32) error {
return client.Timing(stat, value, rate)
}
default:
log.Fatal("Unsupported state type")
}
pertick := opts.Volume / int(opts.Duration.Seconds()) / 10
// add some extra tiem, because the first tick takes a while
ender := time.After(opts.Duration + 100*time.Millisecond)
c := time.Tick(time.Second / 10)
count := 0
for {
select {
case <-c:
for x := 0; x < pertick; x++ {
err := stat(opts.Name, opts.StatValue, opts.Rate)
if err != nil {
log.Printf("Got Error: %+v", err)
break
}
count += 1
}
case <-ender:
log.Printf("%d events called", count)
os.Exit(0)
return
}
}
}

24
vendor/github.com/cihub/seelog/LICENSE.txt generated vendored
View file

@ -1,24 +0,0 @@
Copyright (c) 2012, Cloud Instruments Co., Ltd. <info@cin.io>
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
* Neither the name of the Cloud Instruments Co., Ltd. nor the
names of its contributors may be used to endorse or promote products
derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

113
vendor/github.com/cihub/seelog/README.markdown generated vendored
View file

@ -1,113 +0,0 @@
Seelog
=======
Seelog is a powerful and easy-to-learn logging framework that provides functionality for flexible dispatching, filtering, and formatting log messages.
It is natively written in the [Go](http://golang.org/) programming language.
[![Build Status](https://drone.io/github.com/cihub/seelog/status.png)](https://drone.io/github.com/cihub/seelog/latest)
Features
------------------
* Xml configuring to be able to change logger parameters without recompilation
* Changing configurations on the fly without app restart
* Possibility to set different log configurations for different project files and functions
* Adjustable message formatting
* Simultaneous log output to multiple streams
* Choosing logger priority strategy to minimize performance hit
* Different output writers
* Console writer
* File writer
* Buffered writer (Chunk writer)
* Rolling log writer (Logging with rotation)
* SMTP writer
* Others... (See [Wiki](https://github.com/cihub/seelog/wiki))
* Log message wrappers (JSON, XML, etc.)
* Global variables and functions for easy usage in standalone apps
* Functions for flexible usage in libraries
Quick-start
-----------
```go
package main
import log "github.com/cihub/seelog"
func main() {
defer log.Flush()
log.Info("Hello from Seelog!")
}
```
Installation
------------
If you don't have the Go development environment installed, visit the
[Getting Started](http://golang.org/doc/install.html) document and follow the instructions. Once you're ready, execute the following command:
```
go get -u github.com/cihub/seelog
```
*IMPORTANT*: If you are not using the latest release version of Go, check out this [wiki page](https://github.com/cihub/seelog/wiki/Notes-on-'go-get')
Documentation
---------------
Seelog has github wiki pages, which contain detailed how-tos references: https://github.com/cihub/seelog/wiki
Examples
---------------
Seelog examples can be found here: [seelog-examples](https://github.com/cihub/seelog-examples)
Issues
---------------
Feel free to push issues that could make Seelog better: https://github.com/cihub/seelog/issues
Changelog
---------------
* **v2.5** : Interaction with other systems. Part 2: custom receivers
* Finished custom receivers feature. Check [wiki](https://github.com/cihub/seelog/wiki/custom-receivers)
* Added 'LoggerFromCustomReceiver'
* Added 'LoggerFromWriterWithMinLevelAndFormat'
* Added 'LoggerFromCustomReceiver'
* Added 'LoggerFromParamConfigAs...'
* **v2.4** : Interaction with other systems. Part 1: wrapping seelog
* Added configurable caller stack skip logic
* Added 'SetAdditionalStackDepth' to 'LoggerInterface'
* **v2.3** : Rethinking 'rolling' receiver
* Reimplemented 'rolling' receiver
* Added 'Max rolls' feature for 'rolling' receiver with type='date'
* Fixed 'rolling' receiver issue: renaming on Windows
* **v2.2** : go1.0 compatibility point [go1.0 tag]
* Fixed internal bugs
* Added 'ANSI n [;k]' format identifier: %EscN
* Made current release go1 compatible
* **v2.1** : Some new features
* Rolling receiver archiving option.
* Added format identifier: %Line
* Smtp: added paths to PEM files directories
* Added format identifier: %FuncShort
* Warn, Error and Critical methods now return an error
* **v2.0** : Second major release. BREAKING CHANGES.
* Support of binaries with stripped symbols
* Added log strategy: adaptive
* Critical message now forces Flush()
* Added predefined formats: xml-debug, xml-debug-short, xml, xml-short, json-debug, json-debug-short, json, json-short, debug, debug-short, fast
* Added receiver: conn (network connection writer)
* BREAKING CHANGE: added Tracef, Debugf, Infof, etc. to satisfy the print/printf principle
* Bug fixes
* **v1.0** : Initial release. Features:
* Xml config
* Changing configurations on the fly without app restart
* Contraints and exceptions
* Formatting
* Log strategies: sync, async loop, async timer
* Receivers: buffered, console, file, rolling, smtp

124
vendor/github.com/cihub/seelog/behavior_adaptive_test.go generated vendored
View file

@ -1,124 +0,0 @@
// Copyright (c) 2012 - Cloud Instruments Co., Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package seelog
import (
"bufio"
"bytes"
"fmt"
"io"
"io/ioutil"
"strconv"
"testing"
)
func countSequencedRowsInFile(filePath string) (int64, error) {
bts, err := ioutil.ReadFile(filePath)
if err != nil {
return 0, err
}
bufReader := bufio.NewReader(bytes.NewBuffer(bts))
var gotCounter int64
for {
line, _, bufErr := bufReader.ReadLine()
if bufErr != nil && bufErr != io.EOF {
return 0, bufErr
}
lineString := string(line)
if lineString == "" {
break
}
intVal, atoiErr := strconv.ParseInt(lineString, 10, 64)
if atoiErr != nil {
return 0, atoiErr
}
if intVal != gotCounter {
return 0, fmt.Errorf("wrong order: %d Expected: %d\n", intVal, gotCounter)
}
gotCounter++
}
return gotCounter, nil
}
func Test_Adaptive(t *testing.T) {
fileName := "beh_test_adaptive.log"
count := 100
Current.Close()
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
return
}
defer func() {
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
}
}()
testConfig := `
<seelog type="adaptive" mininterval="1000" maxinterval="1000000" critmsgcount="100">
<outputs formatid="msg">
<file path="` + fileName + `"/>
</outputs>
<formats>
<format id="msg" format="%Msg%n"/>
</formats>
</seelog>`
logger, _ := LoggerFromConfigAsString(testConfig)
err := ReplaceLogger(logger)
if err != nil {
t.Error(err)
return
}
for i := 0; i < count; i++ {
Trace(strconv.Itoa(i))
}
Flush()
gotCount, err := countSequencedRowsInFile(fileName)
if err != nil {
t.Error(err)
return
}
if int64(count) != gotCount {
t.Errorf("wrong count of log messages. Expected: %v, got: %v.", count, gotCount)
return
}
Current.Close()
}

129
vendor/github.com/cihub/seelog/behavior_adaptivelogger.go generated vendored
View file

@ -1,129 +0,0 @@
// Copyright (c) 2012 - Cloud Instruments Co., Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package seelog
import (
"errors"
"fmt"
"math"
"time"
)
var (
adaptiveLoggerMaxInterval = time.Minute
adaptiveLoggerMaxCriticalMsgCount = uint32(1000)
)
// asyncAdaptiveLogger represents asynchronous adaptive logger which acts like
// an async timer logger, but its interval depends on the current message count
// in the queue.
//
// Interval = I, minInterval = m, maxInterval = M, criticalMsgCount = C, msgCount = c:
// I = m + (C - Min(c, C)) / C * (M - m)
type asyncAdaptiveLogger struct {
asyncLogger
minInterval time.Duration
criticalMsgCount uint32
maxInterval time.Duration
}
// newAsyncLoopLogger creates a new asynchronous adaptive logger
func newAsyncAdaptiveLogger(
config *logConfig,
minInterval time.Duration,
maxInterval time.Duration,
criticalMsgCount uint32) (*asyncAdaptiveLogger, error) {
if minInterval <= 0 {
return nil, errors.New("async adaptive logger min interval should be > 0")
}
if maxInterval > adaptiveLoggerMaxInterval {
return nil, fmt.Errorf("async adaptive logger max interval should be <= %s",
adaptiveLoggerMaxInterval)
}
if criticalMsgCount <= 0 {
return nil, errors.New("async adaptive logger critical msg count should be > 0")
}
if criticalMsgCount > adaptiveLoggerMaxCriticalMsgCount {
return nil, fmt.Errorf("async adaptive logger critical msg count should be <= %s",
adaptiveLoggerMaxInterval)
}
asnAdaptiveLogger := new(asyncAdaptiveLogger)
asnAdaptiveLogger.asyncLogger = *newAsyncLogger(config)
asnAdaptiveLogger.minInterval = minInterval
asnAdaptiveLogger.maxInterval = maxInterval
asnAdaptiveLogger.criticalMsgCount = criticalMsgCount
go asnAdaptiveLogger.processQueue()
return asnAdaptiveLogger, nil
}
func (asnAdaptiveLogger *asyncAdaptiveLogger) processItem() (closed bool, itemCount int) {
asnAdaptiveLogger.queueHasElements.L.Lock()
defer asnAdaptiveLogger.queueHasElements.L.Unlock()
for asnAdaptiveLogger.msgQueue.Len() == 0 && !asnAdaptiveLogger.Closed() {
asnAdaptiveLogger.queueHasElements.Wait()
}
if asnAdaptiveLogger.Closed() {
return true, asnAdaptiveLogger.msgQueue.Len()
}
asnAdaptiveLogger.processQueueElement()
return false, asnAdaptiveLogger.msgQueue.Len() - 1
}
// I = m + (C - Min(c, C)) / C * (M - m) =>
// I = m + cDiff * mDiff,
// cDiff = (C - Min(c, C)) / C)
// mDiff = (M - m)
func (asnAdaptiveLogger *asyncAdaptiveLogger) calcAdaptiveInterval(msgCount int) time.Duration {
critCountF := float64(asnAdaptiveLogger.criticalMsgCount)
cDiff := (critCountF - math.Min(float64(msgCount), critCountF)) / critCountF
mDiff := float64(asnAdaptiveLogger.maxInterval - asnAdaptiveLogger.minInterval)
return asnAdaptiveLogger.minInterval + time.Duration(cDiff*mDiff)
}
func (asnAdaptiveLogger *asyncAdaptiveLogger) processQueue() {
for !asnAdaptiveLogger.Closed() {
closed, itemCount := asnAdaptiveLogger.processItem()
if closed {
break
}
interval := asnAdaptiveLogger.calcAdaptiveInterval(itemCount)
<-time.After(interval)
}
}

142
vendor/github.com/cihub/seelog/behavior_asynclogger.go generated vendored
View file

@ -1,142 +0,0 @@
// Copyright (c) 2012 - Cloud Instruments Co., Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package seelog
import (
"container/list"
"fmt"
"sync"
)
// MaxQueueSize is the critical number of messages in the queue that result in an immediate flush.
const (
MaxQueueSize = 10000
)
type msgQueueItem struct {
level LogLevel
context LogContextInterface
message fmt.Stringer
}
// asyncLogger represents common data for all asynchronous loggers
type asyncLogger struct {
commonLogger
msgQueue *list.List
queueHasElements *sync.Cond
}
// newAsyncLogger creates a new asynchronous logger
func newAsyncLogger(config *logConfig) *asyncLogger {
asnLogger := new(asyncLogger)
asnLogger.msgQueue = list.New()
asnLogger.queueHasElements = sync.NewCond(new(sync.Mutex))
asnLogger.commonLogger = *newCommonLogger(config, asnLogger)
return asnLogger
}
func (asnLogger *asyncLogger) innerLog(
level LogLevel,
context LogContextInterface,
message fmt.Stringer) {
asnLogger.addMsgToQueue(level, context, message)
}
func (asnLogger *asyncLogger) Close() {
asnLogger.m.Lock()
defer asnLogger.m.Unlock()
if !asnLogger.Closed() {
asnLogger.flushQueue(true)
asnLogger.config.RootDispatcher.Flush()
if err := asnLogger.config.RootDispatcher.Close(); err != nil {
reportInternalError(err)
}
asnLogger.closedM.Lock()
asnLogger.closed = true
asnLogger.closedM.Unlock()
asnLogger.queueHasElements.Broadcast()
}
}
func (asnLogger *asyncLogger) Flush() {
asnLogger.m.Lock()
defer asnLogger.m.Unlock()
if !asnLogger.Closed() {
asnLogger.flushQueue(true)
asnLogger.config.RootDispatcher.Flush()
}
}
func (asnLogger *asyncLogger) flushQueue(lockNeeded bool) {
if lockNeeded {
asnLogger.queueHasElements.L.Lock()
defer asnLogger.queueHasElements.L.Unlock()
}
for asnLogger.msgQueue.Len() > 0 {
asnLogger.processQueueElement()
}
}
func (asnLogger *asyncLogger) processQueueElement() {
if asnLogger.msgQueue.Len() > 0 {
backElement := asnLogger.msgQueue.Front()
msg, _ := backElement.Value.(msgQueueItem)
asnLogger.processLogMsg(msg.level, msg.message, msg.context)
asnLogger.msgQueue.Remove(backElement)
}
}
func (asnLogger *asyncLogger) addMsgToQueue(
level LogLevel,
context LogContextInterface,
message fmt.Stringer) {
if !asnLogger.Closed() {
asnLogger.queueHasElements.L.Lock()
defer asnLogger.queueHasElements.L.Unlock()
if asnLogger.msgQueue.Len() >= MaxQueueSize {
fmt.Printf("Seelog queue overflow: more than %v messages in the queue. Flushing.\n", MaxQueueSize)
asnLogger.flushQueue(false)
}
queueItem := msgQueueItem{level, context, message}
asnLogger.msgQueue.PushBack(queueItem)
asnLogger.queueHasElements.Broadcast()
} else {
err := fmt.Errorf("queue closed! Cannot process element: %d %#v", level, message)
reportInternalError(err)
}
}

133
vendor/github.com/cihub/seelog/behavior_asyncloop_test.go generated vendored
View file

@ -1,133 +0,0 @@
// Copyright (c) 2012 - Cloud Instruments Co., Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package seelog
import (
"strconv"
"testing"
)
func Test_Asyncloop(t *testing.T) {
fileName := "beh_test_asyncloop.log"
count := 100
Current.Close()
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
return
}
defer func() {
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
}
}()
testConfig := `
<seelog type="asyncloop">
<outputs formatid="msg">
<file path="` + fileName + `"/>
</outputs>
<formats>
<format id="msg" format="%Msg%n"/>
</formats>
</seelog>`
logger, _ := LoggerFromConfigAsString(testConfig)
err := ReplaceLogger(logger)
if err != nil {
t.Error(err)
return
}
for i := 0; i < count; i++ {
Trace(strconv.Itoa(i))
}
Flush()
gotCount, err := countSequencedRowsInFile(fileName)
if err != nil {
t.Error(err)
return
}
if int64(count) != gotCount {
t.Errorf("wrong count of log messages. Expected: %v, got: %v.", count, gotCount)
return
}
Current.Close()
}
func Test_AsyncloopOff(t *testing.T) {
fileName := "beh_test_asyncloopoff.log"
count := 100
Current.Close()
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
return
}
testConfig := `
<seelog type="asyncloop" levels="off">
<outputs formatid="msg">
<file path="` + fileName + `"/>
</outputs>
<formats>
<format id="msg" format="%Msg%n"/>
</formats>
</seelog>`
logger, _ := LoggerFromConfigAsString(testConfig)
err := ReplaceLogger(logger)
if err != nil {
t.Error(err)
return
}
for i := 0; i < count; i++ {
Trace(strconv.Itoa(i))
}
Flush()
ex, err := fileExists(fileName)
if err != nil {
t.Error(err)
}
if ex {
t.Errorf("logger at level OFF is not expected to create log file at all.")
defer func() {
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
}
}()
}
Current.Close()
}

69
vendor/github.com/cihub/seelog/behavior_asynclooplogger.go generated vendored
View file

@ -1,69 +0,0 @@
// Copyright (c) 2012 - Cloud Instruments Co., Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package seelog
// asyncLoopLogger represents asynchronous logger which processes the log queue in
// a 'for' loop
type asyncLoopLogger struct {
asyncLogger
}
// newAsyncLoopLogger creates a new asynchronous loop logger
func newAsyncLoopLogger(config *logConfig) *asyncLoopLogger {
asnLoopLogger := new(asyncLoopLogger)
asnLoopLogger.asyncLogger = *newAsyncLogger(config)
go asnLoopLogger.processQueue()
return asnLoopLogger
}
func (asnLoopLogger *asyncLoopLogger) processItem() (closed bool) {
asnLoopLogger.queueHasElements.L.Lock()
defer asnLoopLogger.queueHasElements.L.Unlock()
for asnLoopLogger.msgQueue.Len() == 0 && !asnLoopLogger.Closed() {
asnLoopLogger.queueHasElements.Wait()
}
if asnLoopLogger.Closed() {
return true
}
asnLoopLogger.processQueueElement()
return false
}
func (asnLoopLogger *asyncLoopLogger) processQueue() {
for !asnLoopLogger.Closed() {
closed := asnLoopLogger.processItem()
if closed {
break
}
}
}

83
vendor/github.com/cihub/seelog/behavior_asynctimer_test.go generated vendored
View file

@ -1,83 +0,0 @@
// Copyright (c) 2012 - Cloud Instruments Co., Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package seelog
import (
"strconv"
"testing"
)
func Test_Asynctimer(t *testing.T) {
fileName := "beh_test_asynctimer.log"
count := 100
Current.Close()
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
return
}
defer func() {
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
}
}()
testConfig := `
<seelog type="asynctimer" asyncinterval="100">
<outputs formatid="msg">
<file path="` + fileName + `"/>
</outputs>
<formats>
<format id="msg" format="%Msg%n"/>
</formats>
</seelog>`
logger, _ := LoggerFromConfigAsString(testConfig)
err := ReplaceLogger(logger)
if err != nil {
t.Error(err)
return
}
for i := 0; i < count; i++ {
Trace(strconv.Itoa(i))
}
Flush()
gotCount, err := countSequencedRowsInFile(fileName)
if err != nil {
t.Error(err)
return
}
if int64(count) != gotCount {
t.Errorf("wrong count of log messages. Expected: %v, got: %v.", count, gotCount)
return
}
Current.Close()
}

82
vendor/github.com/cihub/seelog/behavior_asynctimerlogger.go generated vendored
View file

@ -1,82 +0,0 @@
// Copyright (c) 2012 - Cloud Instruments Co., Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package seelog
import (
"errors"
"time"
)
// asyncTimerLogger represents asynchronous logger which processes the log queue each
// 'duration' nanoseconds
type asyncTimerLogger struct {
asyncLogger
interval time.Duration
}
// newAsyncLoopLogger creates a new asynchronous loop logger
func newAsyncTimerLogger(config *logConfig, interval time.Duration) (*asyncTimerLogger, error) {
if interval <= 0 {
return nil, errors.New("async logger interval should be > 0")
}
asnTimerLogger := new(asyncTimerLogger)
asnTimerLogger.asyncLogger = *newAsyncLogger(config)
asnTimerLogger.interval = interval
go asnTimerLogger.processQueue()
return asnTimerLogger, nil
}
func (asnTimerLogger *asyncTimerLogger) processItem() (closed bool) {
asnTimerLogger.queueHasElements.L.Lock()
defer asnTimerLogger.queueHasElements.L.Unlock()
for asnTimerLogger.msgQueue.Len() == 0 && !asnTimerLogger.Closed() {
asnTimerLogger.queueHasElements.Wait()
}
if asnTimerLogger.Closed() {
return true
}
asnTimerLogger.processQueueElement()
return false
}
func (asnTimerLogger *asyncTimerLogger) processQueue() {
for !asnTimerLogger.Closed() {
closed := asnTimerLogger.processItem()
if closed {
break
}
<-time.After(asnTimerLogger.interval)
}
}

75
vendor/github.com/cihub/seelog/behavior_synclogger.go generated vendored
View file

@ -1,75 +0,0 @@
// Copyright (c) 2012 - Cloud Instruments Co., Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package seelog
import (
"fmt"
)
// syncLogger performs logging in the same goroutine where 'Trace/Debug/...'
// func was called
type syncLogger struct {
commonLogger
}
// newSyncLogger creates a new synchronous logger
func newSyncLogger(config *logConfig) *syncLogger {
syncLogger := new(syncLogger)
syncLogger.commonLogger = *newCommonLogger(config, syncLogger)
return syncLogger
}
func (syncLogger *syncLogger) innerLog(
level LogLevel,
context LogContextInterface,
message fmt.Stringer) {
syncLogger.processLogMsg(level, message, context)
}
func (syncLogger *syncLogger) Close() {
syncLogger.m.Lock()
defer syncLogger.m.Unlock()
if !syncLogger.Closed() {
if err := syncLogger.config.RootDispatcher.Close(); err != nil {
reportInternalError(err)
}
syncLogger.closedM.Lock()
syncLogger.closed = true
syncLogger.closedM.Unlock()
}
}
func (syncLogger *syncLogger) Flush() {
syncLogger.m.Lock()
defer syncLogger.m.Unlock()
if !syncLogger.Closed() {
syncLogger.config.RootDispatcher.Flush()
}
}

81
vendor/github.com/cihub/seelog/behavior_synclogger_test.go generated vendored
View file

@ -1,81 +0,0 @@
// Copyright (c) 2012 - Cloud Instruments Co., Ltd.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation
// and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
// ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package seelog
import (
"strconv"
"testing"
)
func Test_Sync(t *testing.T) {
fileName := "beh_test_sync.log"
count := 100
Current.Close()
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
return
}
defer func() {
if e := tryRemoveFile(fileName); e != nil {
t.Error(e)
}
}()
testConfig := `
<seelog type="sync">
<outputs formatid="msg">
<file path="` + fileName + `"/>
</outputs>
<formats>
<format id="msg" format="%Msg%n"/>
</formats>
</seelog>`
logger, _ := LoggerFromConfigAsString(testConfig)
err := ReplaceLogger(logger)
if err != nil {
t.Error(err)
return
}
for i := 0; i < count; i++ {
Trace(strconv.Itoa(i))
}
gotCount, err := countSequencedRowsInFile(fileName)
if err != nil {
t.Error(err)
return
}
if int64(count) != gotCount {
t.Errorf("wrong count of log messages. Expected: %v, got: %v.", count, gotCount)
return
}
Current.Close()
}

Some files were not shown because too many files have changed in this diff Show more