scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merged

Browse source
This commit is contained in:
Todd Gruben 2016-02-24 14:19:44 -06:00
commit 759ad537be
18 changed files with 588 additions and 457 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
attr.go
View file

@ -112,6 +112,12 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error {
}
switch v := v.(type) {
case int:
attr[k] = uint64(v)
case uint:
attr[k] = uint64(v)
case int64:
attr[k] = uint64(v)
case string, uint64, bool:
attr[k] = v
default:

8
attr_test.go
View file

@ -15,9 +15,9 @@ func TestAttrStore_Attrs(t *testing.T) {
defer s.Close()
// Set attributes.
if err := s.SetAttrs(1, map[string]interface{}{"A": int64(100)}); err != nil {
if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(2, map[string]interface{}{"A": int64(200)}); err != nil {
} else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil {
t.Fatal(err)
} else if err := s.SetAttrs(1, map[string]interface{}{"B": "VALUE"}); err != nil {
t.Fatal(err)
@ -26,14 +26,14 @@ func TestAttrStore_Attrs(t *testing.T) {
// Retrieve attributes for profile #1.
if m, err := s.Attrs(1); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE"}) {
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(100), "B": "VALUE"}) {
t.Fatalf("unexpected attrs(1): %#v", m)
}
// Retrieve attributes for profile #2.
if m, err := s.Attrs(2); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) {
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(200)}) {
t.Fatalf("unexpected attrs(2): %#v", m)
}
}

4
cmd/pilosa/main_test.go
View file

@ -31,10 +31,8 @@ func TestMain_Set_Quick(t *testing.T) {
// Execute SetBit() commands.
for _, cmd := range cmds {
if res, err := m.Query("db=d", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil {
if _, err := m.Query("db=d", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil {
t.Fatal(err)
} else if res != `{}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}

76
executor.go
View file

@ -8,6 +8,7 @@ import (
"net/http"
"net/url"
"sort"
"time"
"github.com/gogo/protobuf/proto"
"github.com/umbel/pilosa/internal"
@ -41,16 +42,21 @@ func NewExecutor(index *Index) *Executor {
func (e *Executor) Index() *Index { return e.index }
// Execute executes a PQL query.
func (e *Executor) Execute(db string, q *pql.Query, slices []uint64) (interface{}, error) {
func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error) {
// Verify that a database is set.
if db == "" {
return nil, ErrDatabaseRequired
}
// Default options.
if opt == nil {
opt = &ExecOptions{}
}
// Ignore slices for set calls.
switch root := q.Root.(type) {
case *pql.SetBit:
return e.executeSetBit(db, root)
return e.executeSetBit(db, root, opt)
case *pql.SetBitmapAttrs:
return nil, e.executeSetBitmapAttrs(db, root)
case *pql.SetProfileAttrs:
@ -70,27 +76,27 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64) (interface{
}
}
return e.executeCall(db, q.Root, slices)
return e.executeCall(db, q.Root, slices, opt)
}
// executeCall executes a call.
func (e *Executor) executeCall(db string, c pql.Call, slices []uint64) (interface{}, error) {
func (e *Executor) executeCall(db string, c pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) {
switch c := c.(type) {
case pql.BitmapCall:
return e.executeBitmapCall(db, c, slices)
return e.executeBitmapCall(db, c, slices, opt)
case *pql.Count:
return e.executeCount(db, c, slices)
return e.executeCount(db, c, slices, opt)
case *pql.Profile:
return e.executeProfile(db, c)
return e.executeProfile(db, c, opt)
case *pql.TopN:
return e.executeTopN(db, c, slices)
return e.executeTopN(db, c, slices, opt)
default:
panic("unreachable")
}
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint64) (*Bitmap, error) {
func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint64, opt *ExecOptions) (*Bitmap, error) {
other := NewBitmap()
for node, nodeSlices := range e.slicesByNode(slices) {
// Execute locally if the hostname matches.
@ -106,7 +112,7 @@ func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint6
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices)
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt)
if err != nil {
return nil, err
}
@ -147,7 +153,7 @@ func (e *Executor) executeBitmapCallSlice(db string, c pql.BitmapCall, slice uin
}
// executeTopN executes a TopN() call.
func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, error) {
func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64, opt *ExecOptions) ([]Pair, error) {
var results []Pair
for node, nodeSlices := range e.slicesByNode(slices) {
// Execute locally if the hostname matches.
@ -163,7 +169,7 @@ func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair,
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices)
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt)
if err != nil {
return nil, err
}
@ -260,7 +266,16 @@ func (e *Executor) executeIntersectSlice(db string, c *pql.Intersect, slice uint
// executeRangeSlice executes a range() call for a local slice.
func (e *Executor) executeRangeSlice(db string, c *pql.Range, slice uint64) (*Bitmap, error) {
panic("FIXME")
frame := c.Frame
if frame == "" {
frame = DefaultFrame
}
f := e.Index().Fragment(db, frame, slice)
if f == nil {
return NewBitmap(), nil
}
return f.Range(c.ID, c.StartTime, c.EndTime), nil
}
// executeUnionSlice executes a union() call for a local slice.
@ -283,7 +298,7 @@ func (e *Executor) executeUnionSlice(db string, c *pql.Union, slice uint64) (*Bi
}
// executeCount executes a count() call.
func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint64, error) {
func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64, opt *ExecOptions) (uint64, error) {
var n uint64
for node, nodeSlices := range e.slicesByNode(slices) {
// Execute locally if the hostname matches.
@ -299,7 +314,7 @@ func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint6
}
// Otherwise execute remotely.
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices)
res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt)
if err != nil {
return 0, err
}
@ -310,12 +325,12 @@ func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint6
// executeProfile executes a Profile() call.
// This call only executes locally since the profile attibutes are stored locally.
func (e *Executor) executeProfile(db string, c *pql.Profile) (*Profile, error) {
func (e *Executor) executeProfile(db string, c *pql.Profile, opt *ExecOptions) (*Profile, error) {
panic("FIXME: impl: e.Index().ProfileAttr(c.ID)")
}
// executeSetBit executes a SetBit() call.
func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) {
func (e *Executor) executeSetBit(db string, c *pql.SetBit, opt *ExecOptions) (bool, error) {
slice := c.ProfileID / SliceWidth
ret := false
for _, node := range e.Cluster.SliceNodes(slice) {
@ -325,7 +340,7 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) {
if err != nil {
return false, fmt.Errorf("fragment: %s", err)
}
val, err := f.SetBit(c.ID, c.ProfileID)
val, err := f.SetBit(c.ID, c.ProfileID, opt.Timestamp, opt.Quantum)
if err != nil {
return false, err
}
@ -336,7 +351,7 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) {
}
// Forward call to remote node otherwise.
if res, err := e.exec(node, db, &pql.Query{Root: c}, nil); err != nil {
if res, err := e.exec(node, db, &pql.Query{Root: c}, nil, opt); err != nil {
return false, err
} else {
ret = res.(bool)
@ -382,13 +397,18 @@ func (e *Executor) executeSetProfileAttrs(db string, c *pql.SetProfileAttrs) err
}
// exec executes a PQL query remotely for a set of slices on a node.
func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64) (result interface{}, err error) {
func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (result interface{}, err error) {
// Encode request object.
buf, err := proto.Marshal(&internal.QueryRequest{
DB: proto.String(db),
Query: proto.String(q.String()),
Slices: slices,
})
pbreq := &internal.QueryRequest{
DB: proto.String(db),
Query: proto.String(q.String()),
Slices: slices,
Quantum: proto.Uint32(uint32(opt.Quantum)),
}
if opt.Timestamp != nil {
pbreq.Timestamp = proto.Int64(opt.Timestamp.UnixNano())
}
buf, err := proto.Marshal(pbreq)
if err != nil {
return nil, err
}
@ -465,6 +485,12 @@ func (e *Executor) slicesByNode(slices []uint64) map[*Node][]uint64 {
return m
}
// ExecOptions represents an execution context for a single Execute() call.
type ExecOptions struct {
Timestamp *time.Time
Quantum TimeQuantum
}
// decodeError returns an error representation of s if s is non-blank.
// Returns nil if s is blank.
func decodeError(s string) error {

102
executor_test.go
View file

@ -17,12 +17,12 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3)
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1)
if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": 123}); err != nil {
if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
t.Fatal(err)
}
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil); err != nil {
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
@ -30,7 +30,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0]))
} else if chunks[1].Value[0] != 2 {
t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1]))
} else if attrs := res.(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": 123}) {
} else if attrs := res.(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
}
@ -45,7 +45,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil {
if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 1 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
@ -67,7 +67,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil {
if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
@ -90,7 +90,7 @@ func TestExecutor_Execute_Union(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil {
if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
@ -110,7 +110,7 @@ func TestExecutor_Execute_Count(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2)
e := NewExecutor(idx.Index, NewCluster(1))
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil {
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if n != uint64(3) {
t.Fatalf("unexpected n: %d", n)
@ -156,23 +156,23 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
// Set two fields on f/10.
// Also set fields on other bitmaps and frames to test isolation.
e := NewExecutor(idx.Index, NewCluster(1))
if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil); err != nil {
if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil); err != nil {
if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=XXX, YYY=1)`), nil); err != nil {
if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=XXX, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil); err != nil {
if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
}
f := idx.Frame("d", "f")
if m, err := f.BitmapAttrStore().Attrs(10); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": uint64(123), "bat": true}) {
t.Fatalf("unexpected bitmap attr: %#v", m)
}
}
@ -183,19 +183,19 @@ func TestExecutor_Execute_TopN(t *testing.T) {
defer idx.Close()
// Set bits for bitmaps 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0, nil, 0)
// Execute query.
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil); err != nil {
if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
{Key: 0, Count: 5},
@ -211,23 +211,23 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
defer idx.Close()
// Set bits for bitmaps 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2, nil, 0)
// Create an intersecting bitmap.
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2)
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1, nil, 0)
idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2, nil, 0)
// Execute query.
e := NewExecutor(idx.Index, NewCluster(1))
if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil); err != nil {
if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []pilosa.Pair{
{Key: 20, Count: 3},
@ -238,6 +238,24 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
}
}
// Ensure a range query can be executed.
func TestExecutor_Execute_Range(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
f := idx.MustCreateFragmentIfNotExists("d", "f.t", 0)
if _, err := f.SetBit(1, 100, MustParseTime("2000-01-01 00:00"), pilosa.YMD); err != nil {
t.Fatal(err)
}
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute("d", MustParse(`Range(id=1, frame=f.t, start="2000-01-01T00:00", end="2000-01-01T01:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res.(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) {
t.Fatalf("unexpected bits: %+v", bits)
}
}
// Ensure a remote query can return a bitmap.
func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c := NewCluster(2)
@ -248,7 +266,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Bitmap(id=10, frame=f)` {
@ -273,7 +291,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1)
e := NewExecutor(idx.Index, c)
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil); err != nil {
if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 3 {
t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks))
@ -294,7 +312,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to return a count.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return uint64(10), nil
}
@ -305,7 +323,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2)
e := NewExecutor(idx.Index, c)
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil {
if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if n != uint64(12) {
t.Fatalf("unexpected n: %d", n)
@ -324,7 +342,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Mock secondary server's executor to verify arguments.
var remoteCalled bool
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `SetBit(id=10, frame=f, profileID=2)` {
@ -339,7 +357,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
defer idx.Close()
e := NewExecutor(idx.Index, c)
if _, err := e.Execute("d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil); err != nil {
if _, err := e.Execute("d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -362,7 +380,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
c.Nodes[1].Host = s.Host()
// Mock secondary server's executor to verify arguments and return a bitmap.
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `TopN(frame=f, n=3)` {
@ -386,7 +404,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", 3).MustSetBits(30, (3*SliceWidth)+2)
e := NewExecutor(idx.Index, c)
if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil); err != nil {
if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res, []pilosa.Pair{
{Key: 0, Count: 5},

33
fragment.go
View file

@ -188,7 +188,7 @@ func (f *Fragment) openStorage() error {
// openCache initializes the cache from bitmap ids persisted to disk.
func (f *Fragment) openCache() error {
// Determine cache type from frame name.
if strings.HasSuffix(f.frame, ".n") {
if strings.HasSuffix(f.frame, FrameSuffixRank) {
c := NewRankCache()
c.ThresholdLength = 50000
c.ThresholdIndex = 45000
@ -312,13 +312,19 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap {
// SetBit sets a bit for a given profile & bitmap within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) {
func (f *Fragment) SetBit(bitmapID, profileID uint64, t *time.Time, q TimeQuantum) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
// Set time bits if this is a time-frame and a timestamp is specified.
if strings.HasSuffix(f.frame, FrameSuffixTime) && t != nil {
return f.setTimeBit(bitmapID, profileID, *t, q)
}
return f.setBit(bitmapID, profileID)
}
func (f *Fragment) setBit(bitmapID, profileID uint64) (bool, error) {
func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) {
// Determine the position of the bit in the storage.
ret := false
pos, err := f.pos(bitmapID, profileID)
@ -340,6 +346,17 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (bool, error) {
}
func (f *Fragment) setTimeBit(bitmapID, profileID uint64, t time.Time, q TimeQuantum) (changed bool, err error) {
for _, timeID := range TimeIDsFromQuantum(q, t, bitmapID) {
if v, err := f.setBit(timeID, profileID); err != nil {
return changed, fmt.Errorf("set time bit: t=%s, q=%s, err=%s", t, q, err)
} else if v {
changed = true
}
}
return changed, nil
}
// ClearBit clears a bit for a given profile & bitmap within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) {
@ -481,16 +498,18 @@ func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap {
f.mu.Lock()
defer f.mu.Unlock()
bitmapIDs := GetRange(start, end, bitmapID)
// Retrieve a list of bitmap ids for a given time range.
bitmapIDs := TimeIDsFromRange(start, end, bitmapID)
if len(bitmapIDs) == 0 {
return NewBitmap()
}
result := f.bitmap(bitmapIDs[0])
// Union all bitmap ids from the time range.
bm := f.bitmap(bitmapIDs[0])
for _, id := range bitmapIDs[1:] {
result = result.Union(f.bitmap(id))
bm = bm.Union(f.bitmap(id))
}
return result
return bm
}
// Import bulk imports a set of bits and then snapshots the storage.

31
fragment_test.go
View file

@ -19,11 +19,11 @@ func TestFragment_SetBit(t *testing.T) {
defer f.Close()
// Set bits on the fragment.
if _, err := f.SetBit(120, 1); err != nil {
if _, err := f.SetBit(120, 1, nil, 0); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 6); err != nil {
} else if _, err := f.SetBit(120, 6, nil, 0); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(121, 0); err != nil {
} else if _, err := f.SetBit(121, 0, nil, 0); err != nil {
t.Fatal(err)
}
@ -50,9 +50,9 @@ func TestFragment_ClearBit(t *testing.T) {
defer f.Close()
// Set and then clear bits on the fragment.
if _, err := f.SetBit(1000, 1); err != nil {
if _, err := f.SetBit(1000, 1, nil, 0); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(1000, 2); err != nil {
} else if _, err := f.SetBit(1000, 2, nil, 0); err != nil {
t.Fatal(err)
} else if _, err := f.ClearBit(1000, 1); err != nil {
t.Fatal(err)
@ -77,9 +77,9 @@ func TestFragment_Snapshot(t *testing.T) {
defer f.Close()
// Set and then clear bits on the fragment.
if _, err := f.SetBit(1000, 1); err != nil {
if _, err := f.SetBit(1000, 1, nil, 0); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(1000, 2); err != nil {
} else if _, err := f.SetBit(1000, 2, nil, 0); err != nil {
t.Fatal(err)
} else if _, err := f.ClearBit(1000, 1); err != nil {
t.Fatal(err)
@ -133,11 +133,11 @@ func TestFragment_TopN_Filter(t *testing.T) {
f.MustSetBits(102, 1, 2)
// Assign attributes.
f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": 10})
f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": 20})
f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)})
f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)})
// Retrieve top bitmaps.
if pairs, err := f.TopN(2, nil, "x", []interface{}{10, 15, 20}); err != nil {
if pairs, err := f.TopN(2, nil, "x", []interface{}{uint64(10), uint64(15), uint64(20)}); err != nil {
t.Fatal(err)
} else if len(pairs) != 2 {
t.Fatalf("unexpected count: %d", len(pairs))
@ -176,6 +176,10 @@ func TestFragment_TopN_Intersect(t *testing.T) {
// Ensure a fragment can return top bitmaps that have many bits set.
func TestFragment_TopN_Intersect_Large(t *testing.T) {
if testing.Short() {
t.Skip("short mode")
}
f := MustOpenFragment("d", "f", 0)
defer f.Close()
@ -218,7 +222,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
// Set bits on the fragment.
for i := uint64(0); i < 1000; i++ {
if _, err := f.SetBit(i, 0); err != nil {
if _, err := f.SetBit(i, 0, nil, 0); err != nil {
t.Fatal(err)
}
}
@ -250,7 +254,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
// Set bits on the fragment.
for i := uint64(0); i < 1000; i++ {
if _, err := f.SetBit(i, 0); err != nil {
if _, err := f.SetBit(i, 0, nil, 0); err != nil {
t.Fatal(err)
}
}
@ -330,9 +334,10 @@ func (f *Fragment) Reopen() error {
}
// MustSetBits sets bits on a bitmap. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) {
for _, profileID := range profileIDs {
if _, err := f.SetBit(bitmapID, profileID); err != nil {
if _, err := f.SetBit(bitmapID, profileID, nil, 0); err != nil {
panic(err)
}
}

8
frame.go
View file

@ -8,6 +8,14 @@ import (
"sync"
)
const (
// FrameSuffixTime is the suffix used for time-based frames.
FrameSuffixTime = ".t"
// FrameSuffixRank is the suffix used for rank-based frames.
FrameSuffixRank = ".n"
)
// Frame represents a container for fragments.
type Frame struct {
mu sync.Mutex

61
handler.go
View file

@ -29,7 +29,7 @@ type Handler struct {
// The execution engine for running queries.
Executor interface {
Execute(db string, query *pql.Query, slices []uint64) (interface{}, error)
Execute(db string, query *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error)
}
// The version to report on the /version endpoint.
@ -95,6 +95,12 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
return
}
// Build execution options.
opt := &ExecOptions{
Timestamp: req.Timestamp,
Quantum: req.Quantum,
}
// Parse query string.
q, err := pql.NewParser(strings.NewReader(req.Query)).Parse()
if err != nil {
@ -104,7 +110,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
}
// Execute the query.
result, err := h.Executor.Execute(req.DB, q, req.Slices)
result, err := h.Executor.Execute(req.DB, q, req.Slices, opt)
resp := &QueryResponse{Result: result, Err: err}
// Fill profile attributes if requested.
@ -196,11 +202,38 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
return nil, errors.New("invalid slice argument")
}
// Parse timestamp, if available.
var timestamp *time.Time
if v := q.Get("timestamp"); v != "" {
layout := "2006-01-02 15:04:05"
if strings.Contains(v, "T") {
layout = "2006-01-02T15:04:05"
}
t, err := time.Parse(layout, v)
if err != nil {
return nil, errors.New("invalid timestamp")
}
timestamp = &t
}
// Parse time granularity.
quantum := YMDH
if s := q.Get("time_granularity"); s != "" {
v, err := ParseTimeQuantum(s)
if err != nil {
return nil, errors.New("invalid time granularity")
}
quantum = v
}
return &QueryRequest{
DB: q.Get("db"),
Query: query,
Slices: slices,
Profiles: q.Get("profiles") == "true",
DB: q.Get("db"),
Query: query,
Slices: slices,
Profiles: q.Get("profiles") == "true",
Timestamp: timestamp,
Quantum: quantum,
}, nil
}
@ -352,15 +385,29 @@ type QueryRequest struct {
// Return profile attributes, if true.
Profiles bool
// Timestamp passed into the query.
Timestamp *time.Time
// Time granularity to use with the timestamp.
Quantum TimeQuantum
}
func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest {
return &QueryRequest{
req := &QueryRequest{
DB: pb.GetDB(),
Query: pb.GetQuery(),
Slices: pb.GetSlices(),
Profiles: pb.GetProfiles(),
Quantum: TimeQuantum(pb.GetQuantum()),
}
if pb.Timestamp != nil {
t := time.Unix(0, pb.GetTimestamp())
req.Timestamp = &t
}
return req
}
// QueryResponse represent a response from a processed query.

30
handler_test.go
View file

@ -29,7 +29,7 @@ func TestHandler_NotFound(t *testing.T) {
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Count(Bitmap(id=100))` {
@ -52,7 +52,7 @@ func TestHandler_Query_Args_URL(t *testing.T) {
// Ensure the handler can accept arguments via protobufs.
func TestHandler_Query_Args_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
if db != "db0" {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `Count(Bitmap(id=100))` {
@ -98,7 +98,7 @@ func TestHandler_Query_Args_Err(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as JSON.
func TestHandler_Query_Uint64_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return uint64(100), nil
}
@ -114,7 +114,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as protobufs.
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return uint64(100), nil
}
@ -137,7 +137,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap as JSON.
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return bm, nil
@ -169,7 +169,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
h := NewHandler()
h.Index = idx.Index
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return bm, nil
@ -187,7 +187,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap as protobuf.
func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return bm, nil
@ -232,7 +232,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
h := NewHandler()
h.Index = idx.Index
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return bm, nil
@ -287,7 +287,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as JSON.
func TestHandler_Query_Pairs_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return []pilosa.Pair{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
@ -306,7 +306,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as protobuf.
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return []pilosa.Pair{
{Key: 1, Count: 2},
{Key: 3, Count: 4},
@ -332,7 +332,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
// Ensure the handler can return an error as JSON.
func TestHandler_Query_Err_JSON(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return nil, errors.New("marker")
}
@ -348,7 +348,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
// Ensure the handler can return an error as protobuf.
func TestHandler_Query_Err_Protobuf(t *testing.T) {
h := NewHandler()
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) {
h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return nil, errors.New("marker")
}
@ -449,13 +449,13 @@ func NewHandler() *Handler {
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(db string, query *pql.Query, slices []uint64) (interface{}, error)
ExecuteFn func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64) (interface{}, error) {
return c.ExecuteFn(db, query, slices)
func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) {
return c.ExecuteFn(db, query, slices, opt)
}
// Server represents a test wrapper for httptest.Server.

73
internal/internal.pb.go
View file

@ -25,7 +25,6 @@ It has these top-level messages:
package internal
import proto "github.com/gogo/protobuf/proto"
import fmt "fmt"
import math "math"
@ -222,6 +221,8 @@ type QueryRequest struct {
Query *string `protobuf:"bytes,2,req,name=Query" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,3,rep,name=Slices" json:"Slices,omitempty"`
Profiles *bool `protobuf:"varint,4,opt,name=Profiles" json:"Profiles,omitempty"`
Timestamp *int64 `protobuf:"varint,5,opt,name=Timestamp" json:"Timestamp,omitempty"`
Quantum *uint32 `protobuf:"varint,6,opt,name=Quantum" json:"Quantum,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
@ -258,6 +259,20 @@ func (m *QueryRequest) GetProfiles() bool {
return false
}
func (m *QueryRequest) GetTimestamp() int64 {
if m != nil && m.Timestamp != nil {
return *m.Timestamp
}
return 0
}
func (m *QueryRequest) GetQuantum() uint32 {
if m != nil && m.Quantum != nil {
return *m.Quantum
}
return 0
}
type QueryResponse struct {
Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"`
Bitmap *Bitmap `protobuf:"bytes,2,opt,name=Bitmap" json:"Bitmap,omitempty"`
@ -414,31 +429,33 @@ func init() {
}
var fileDescriptor0 = []byte{
// 410 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x84, 0x92, 0xc1, 0x8e, 0xda, 0x30,
0x10, 0x86, 0x15, 0xe2, 0x04, 0x32, 0x29, 0x01, 0xdc, 0x0b, 0xaa, 0x84, 0x8a, 0xcc, 0x05, 0xf5,
0xc0, 0x01, 0xf5, 0x05, 0x0a, 0xb4, 0x2a, 0x42, 0x45, 0xb4, 0xa8, 0x3d, 0x37, 0xa2, 0x2e, 0x44,
0x0d, 0x71, 0xea, 0x38, 0x07, 0x5e, 0xa6, 0xcf, 0xda, 0xb1, 0xe3, 0x04, 0x76, 0x97, 0xd5, 0x9e,
0x22, 0xff, 0x1e, 0xcf, 0x7c, 0xff, 0xfc, 0x81, 0x28, 0xc9, 0x14, 0x97, 0x59, 0x9c, 0xce, 0x72,
0x29, 0x94, 0xa0, 0x9d, 0xfa, 0xcc, 0x3e, 0x83, 0xbf, 0x48, 0xd4, 0x39, 0xce, 0xe9, 0x5b, 0xf0,
0x97, 0xa7, 0x32, 0xfb, 0x53, 0x0c, 0x9d, 0xb1, 0x3b, 0x0d, 0xe7, 0xbd, 0x59, 0xf3, 0xc8, 0xe8,
0x74, 0x04, 0xde, 0x07, 0xa5, 0x64, 0x31, 0x6c, 0x99, 0xfb, 0xe8, 0x7a, 0xaf, 0x65, 0x36, 0x01,
0xaf, 0xaa, 0x0b, 0xc1, 0xdd, 0xf0, 0x0b, 0x76, 0x69, 0x4d, 0x09, 0xed, 0x82, 0xf7, 0x23, 0x4e,
0x4b, 0x6e, 0x1e, 0x11, 0xc6, 0x80, 0xec, 0xe2, 0x44, 0x3e, 0xa9, 0x59, 0x8a, 0x32, 0x53, 0x58,
0x83, 0x47, 0xf6, 0x0e, 0x5c, 0x44, 0xa2, 0x7d, 0xe8, 0x54, 0x64, 0xeb, 0x95, 0xad, 0x1b, 0x40,
0xb0, 0x93, 0xe2, 0x77, 0x92, 0x72, 0x94, 0xaa, 0xda, 0xf7, 0xd0, 0xb6, 0x12, 0x05, 0x68, 0x35,
0x95, 0x2f, 0xa0, 0x6e, 0x81, 0xe8, 0xef, 0x2d, 0x45, 0x40, 0x5f, 0x43, 0xb8, 0x57, 0x32, 0xc9,
0x8e, 0x35, 0xaf, 0x83, 0x22, 0x8e, 0xfc, 0x8e, 0x6f, 0x2b, 0xc9, 0x45, 0xc9, 0x50, 0x2c, 0x84,
0x48, 0x2b, 0x89, 0xa0, 0xd4, 0x61, 0x53, 0x68, 0xeb, 0x7e, 0x5f, 0x70, 0x8b, 0xcd, 0x64, 0xe7,
0xee, 0xe4, 0x0d, 0xbc, 0xfa, 0x5a, 0x72, 0x79, 0xf9, 0xc6, 0xff, 0x96, 0xbc, 0x50, 0x1a, 0x7a,
0xb5, 0xb0, 0x00, 0xb8, 0x06, 0x73, 0x67, 0xac, 0x05, 0x34, 0x02, 0x7f, 0x9f, 0x26, 0x07, 0x5e,
0xe0, 0x5c, 0x5c, 0x9d, 0xde, 0x87, 0xb5, 0x5a, 0xd8, 0xb1, 0xff, 0x1c, 0xe8, 0xda, 0x6e, 0x45,
0x2e, 0xb2, 0x82, 0x6b, 0x43, 0x1f, 0xa5, 0xc4, 0x7e, 0x9a, 0x7d, 0x5c, 0x47, 0x6b, 0xbc, 0x84,
0xf3, 0xfe, 0x95, 0xc5, 0x46, 0x1e, 0x80, 0xb3, 0xb5, 0xae, 0x90, 0x5b, 0x07, 0xa3, 0x5b, 0x3f,
0xe2, 0x36, 0x79, 0x4d, 0x6e, 0x86, 0x7b, 0xa6, 0x62, 0x70, 0x53, 0x61, 0x13, 0xe8, 0x41, 0x7b,
0x79, 0x8a, 0xb3, 0x23, 0xff, 0x35, 0xf4, 0x0d, 0xe0, 0x4f, 0xe8, 0xae, 0xcf, 0xb9, 0x90, 0xea,
0x19, 0xbb, 0x9f, 0x64, 0x7c, 0xe6, 0xd6, 0x2e, 0x1e, 0x8d, 0x5d, 0xe4, 0xb1, 0x59, 0xd7, 0xe9,
0x57, 0x4c, 0x84, 0xe2, 0xeb, 0x26, 0xfe, 0x8a, 0x82, 0xb0, 0x11, 0x44, 0xf5, 0x84, 0x3b, 0x2b,
0x60, 0x6f, 0xf0, 0xcf, 0x8a, 0x0f, 0x27, 0xfe, 0xb0, 0x9d, 0x8e, 0x86, 0xfc, 0x0f, 0x00, 0x00,
0xff, 0xff, 0x98, 0x36, 0x6d, 0xac, 0x14, 0x03, 0x00, 0x00,
// 436 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x84, 0x92, 0xcf, 0x6e, 0xd4, 0x30,
0x10, 0xc6, 0x95, 0x8d, 0x93, 0x4d, 0x26, 0x24, 0x6d, 0xcd, 0x25, 0x42, 0xaa, 0xa8, 0xdc, 0xcb,
0x8a, 0x43, 0x0f, 0x15, 0x2f, 0xc0, 0x6e, 0x41, 0x54, 0x88, 0xaa, 0xa5, 0xc0, 0x19, 0xab, 0x98,
0x6e, 0x44, 0x62, 0x07, 0xc7, 0x39, 0xf4, 0x65, 0x78, 0x56, 0xc6, 0x7f, 0x92, 0x2e, 0xb0, 0x88,
0x53, 0x94, 0xcf, 0x63, 0x7f, 0xbf, 0xf9, 0x66, 0xa0, 0x6a, 0xa4, 0x11, 0x5a, 0xf2, 0xf6, 0xac,
0xd7, 0xca, 0x28, 0x9a, 0x4d, 0xff, 0xec, 0x2d, 0xa4, 0xeb, 0xc6, 0x74, 0xbc, 0xa7, 0xcf, 0x21,
0xdd, 0x6c, 0x47, 0xf9, 0x7d, 0xa8, 0xa3, 0x93, 0x78, 0x55, 0x9c, 0x1f, 0x9c, 0xcd, 0x97, 0x9c,
0x4e, 0x8f, 0x21, 0x79, 0x65, 0x8c, 0x1e, 0xea, 0x85, 0x3b, 0xaf, 0x1e, 0xcf, 0xad, 0xcc, 0x4e,
0x21, 0xf1, 0x75, 0x05, 0xc4, 0xef, 0xc4, 0x03, 0xbe, 0xb2, 0x58, 0x11, 0x5a, 0x42, 0xf2, 0x99,
0xb7, 0xa3, 0x70, 0x97, 0x08, 0x63, 0x40, 0xae, 0x79, 0xa3, 0xff, 0xaa, 0xd9, 0xa8, 0x51, 0x1a,
0xac, 0xc1, 0x5f, 0xf6, 0x02, 0x62, 0x44, 0xa2, 0x87, 0x90, 0x79, 0xb2, 0xcb, 0x8b, 0x50, 0x77,
0x04, 0xf9, 0xb5, 0x56, 0xdf, 0x9a, 0x56, 0xa0, 0xe4, 0x6b, 0x5f, 0xc2, 0x32, 0x48, 0x14, 0x60,
0x31, 0x57, 0xfe, 0x07, 0xf5, 0x0a, 0x88, 0xfd, 0xee, 0x52, 0xe4, 0xf4, 0x29, 0x14, 0xb7, 0x46,
0x37, 0xf2, 0x7e, 0xe2, 0x8d, 0x50, 0x44, 0xcb, 0x4f, 0x78, 0xd7, 0x4b, 0x31, 0x4a, 0x8e, 0x62,
0xad, 0x54, 0xeb, 0x25, 0x82, 0x52, 0xc6, 0x56, 0xb0, 0xb4, 0xef, 0xbd, 0xc7, 0x14, 0x67, 0xe7,
0x68, 0xaf, 0xb3, 0x82, 0x27, 0x37, 0xa3, 0xd0, 0x0f, 0x1f, 0xc4, 0x8f, 0x51, 0x0c, 0xc6, 0x42,
0x5f, 0xac, 0x03, 0x00, 0xc6, 0xe0, 0xce, 0x5c, 0x6b, 0x39, 0xad, 0x20, 0xbd, 0x6d, 0x9b, 0x3b,
0x31, 0xa0, 0x2f, 0x46, 0x67, 0xf3, 0x08, 0xad, 0x0e, 0xde, 0xd6, 0x92, 0x7c, 0x6c, 0x3a, 0x7c,
0x86, 0x77, 0x7d, 0x9d, 0xa0, 0x14, 0xd3, 0x03, 0x58, 0xde, 0x8c, 0x5c, 0x9a, 0xb1, 0xab, 0x53,
0x14, 0x4a, 0xf6, 0x33, 0x82, 0x32, 0x38, 0x0e, 0xbd, 0x92, 0x83, 0xb0, 0x4d, 0xbf, 0xd6, 0x1a,
0x3d, 0x6d, 0x7f, 0x27, 0xd3, 0xf8, 0x5d, 0xbf, 0xc5, 0xf9, 0xe1, 0x23, 0x6f, 0x58, 0x8b, 0x1c,
0xa2, 0xab, 0xd0, 0x39, 0xf6, 0x66, 0x87, 0x67, 0xed, 0xff, 0xe8, 0xcd, 0xcd, 0xf4, 0x74, 0x07,
0x30, 0x71, 0x15, 0x47, 0x3b, 0x15, 0x61, 0x4a, 0x08, 0xb8, 0xd9, 0x72, 0x79, 0x2f, 0xbe, 0x3a,
0xc0, 0x8c, 0x7d, 0x81, 0xf2, 0xb2, 0xeb, 0x95, 0x36, 0xff, 0x88, 0xe4, 0x8d, 0xe6, 0x9d, 0x08,
0x91, 0xe0, 0xaf, 0x8b, 0x04, 0x79, 0xc2, 0x3e, 0x4c, 0x1b, 0xe2, 0x99, 0x08, 0xc5, 0xdb, 0xf3,
0x8a, 0x78, 0x0a, 0xc2, 0x8e, 0xa1, 0x9a, 0x1c, 0xf6, 0x44, 0xc0, 0x9e, 0xe1, 0xf6, 0xf1, 0xbb,
0xad, 0xf8, 0xfd, 0x39, 0x3b, 0x3e, 0xf2, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x77, 0xe5, 0x8e, 0x3b,
0x38, 0x03, 0x00, 0x00,
}

10
internal/internal.proto
View file

@ -37,10 +37,12 @@ message AttrMap {
}
message QueryRequest {
required string DB = 1;
required string Query = 2;
repeated uint64 Slices = 3;
optional bool Profiles = 4;
required string DB = 1;
required string Query = 2;
repeated uint64 Slices = 3;
optional bool Profiles = 4;
optional int64 Timestamp = 5;
optional uint32 Quantum = 6;
}
message QueryResponse {

2
pql/parser.go
View file

@ -441,7 +441,7 @@ func (p *Parser) parseSetProfileAttrsCall() (*SetProfileAttrs, error) {
case string, bool:
c.Attrs[key] = v
case uint64:
c.Attrs[key] = int64(v)
c.Attrs[key] = v
default:
return nil, parseErrorf(pos, "invalid SetProfileAttrs() arg: %v", arg.key)
}

4
pql/parser_test.go
View file

@ -222,7 +222,7 @@ func TestParser_Parse_SetBitmapAttrs_Key(t *testing.T) {
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": int64(123),
"bar": uint64(123),
"baz": true,
"bat": false,
"x": nil,
@ -244,7 +244,7 @@ func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) {
Frame: "b.n",
Attrs: map[string]interface{}{
"foo": "bar",
"bar": int64(123),
"bar": uint64(123),
},
},
}) {

157
time.go Normal file
View file

@ -0,0 +1,157 @@
package pilosa
import (
"errors"
"strings"
"time"
)
// TimeQuantum represents a time granularity for time-based bitmap ids.
type TimeQuantum uint
// ParseTimeQuantum parses s into a quantum.
func ParseTimeQuantum(s string) (TimeQuantum, error) {
switch strings.ToUpper(s) {
case "Y":
return Y, nil
case "M":
return YM, nil
case "D":
return YMD, nil
case "H":
return YMDH, nil
default:
return 0, errors.New("invalid quantum")
}
}
// String returns the string representation of the quantum.
func (q TimeQuantum) String() string {
switch q {
case Y:
return "Y"
case YM:
return "M"
case YMD:
return "D"
case YMDH:
return "H"
default:
return "Y"
}
}
const (
Y TimeQuantum = 3
YM TimeQuantum = 2
YMD TimeQuantum = 1
YMDH TimeQuantum = 0
)
// TimeID returns a packed time identifier from a year/month/day/hour & tile.
func TimeID(q TimeQuantum, year uint, month uint, day uint, hour uint, tileID uint64) uint64 {
v := uint64((uint(q) << 30) | ((year - 1970) << 23) | (month << 19) | (day << 14) | (hour << 9))
return (v << 32) | tileID
}
// TimeIDsFromRange returns timestamp bitmap ids within a time range.
func TimeIDsFromRange(start, end time.Time, tileID uint64) []uint64 {
t := start
var results []uint64
for t.Before(end) {
if !nextDay(t, end) {
break
}
if t.Hour() == 0 {
if !nextMonth(t, end) {
break
}
if t.Day() == 1 {
if !nextYear(t, end) {
break
}
if t.Month() == 1 {
break
}
results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID))
t = t.AddDate(0, 1, 0)
} else {
results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID))
t = t.AddDate(0, 0, 1)
}
} else {
results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID))
t = t.Add(time.Hour)
}
}
for t.Before(end) {
if nextYear(t, end) {
results = append(results, TimeID(Y, uint(t.Year()), 0, 0, 0, tileID))
t = t.AddDate(1, 0, 0)
} else if nextMonth(t, end) {
results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID))
t = t.AddDate(0, 1, 0)
} else if nextDay(t, end) {
results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID))
t = t.AddDate(0, 0, 1)
} else {
results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID))
t = t.Add(time.Hour)
}
}
return results
}
func nextYear(start time.Time, end time.Time) bool {
next := start.AddDate(1, 0, 0)
if next.Year() == end.Year() {
return true
}
return end.After(next)
}
func nextMonth(start time.Time, end time.Time) bool {
next := start.AddDate(0, 1, 0)
y1, m1, _ := next.Date()
y2, m2, _ := end.Date()
if (y1 == y2) && (m1 == m2) {
return true
}
return end.After(next)
}
func nextDay(start time.Time, end time.Time) bool {
next := start.AddDate(0, 0, 1)
y1, m1, d1 := next.Date()
y2, m2, d2 := end.Date()
if (y1 == y2) && (m1 == m2) && (d1 == d2) {
return true
}
return end.After(next)
}
// TimeIDsFromQuantum returns a list of time identifiers for a single quantum.
func TimeIDsFromQuantum(q TimeQuantum, t time.Time, tileID uint64) []uint64 {
y, m, d, h := uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour())
v := make([]uint64, 0, 4)
if q <= Y {
v = append(v, TimeID(Y, y, 0, 0, 0, tileID))
}
if q <= YM {
v = append(v, TimeID(YM, y, m, 0, 0, tileID))
}
if q <= YMD {
v = append(v, TimeID(YMD, y, m, d, 0, tileID))
}
if q <= YMDH {
v = append(v, TimeID(YMDH, y, m, d, h, tileID))
}
return v
}

134
time_test.go Normal file
View file

@ -0,0 +1,134 @@
package pilosa_test
import (
"testing"
"time"
"github.com/umbel/pilosa"
)
func TestTimeIDsFromRange_1h_0(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-08-11 14:00"),
*MustParseTime("2014-08-11 16:00"),
uint64(1),
); len(m) != 2 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_1h_1(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-02 10:03"),
*MustParseTime("2014-01-02 11:03"),
uint64(1),
); len(m) != 1 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_2h(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-02 10:03"),
*MustParseTime("2014-01-02 12:03"),
uint64(1),
); len(m) != 2 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_24h(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-02 12:03"),
*MustParseTime("2014-01-03 12:03"),
uint64(1),
); len(m) != 24 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_1d(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-02 00:00"),
*MustParseTime("2014-01-03 00:00"),
uint64(1),
); len(m) != 1 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_1d1h(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-02 00:00"),
*MustParseTime("2014-01-03 01:00"),
uint64(1),
); len(m) != 2 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_1h1d(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-02 23:00"),
*MustParseTime("2014-01-04 00:00"),
uint64(1),
); len(m) != 2 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_1h1d1h(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-02 23:00"),
*MustParseTime("2014-01-04 01:00"),
uint64(1),
); len(m) != 3 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_1y(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-01 00:00"),
*MustParseTime("2015-01-01 00:00"),
uint64(1),
); len(m) != 1 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_1h1d1m(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-30 23:00"),
*MustParseTime("2014-03-01 00:00"),
uint64(1),
); len(m) != 3 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromRange_1h1d1m1d1h(t *testing.T) {
if m := pilosa.TimeIDsFromRange(
*MustParseTime("2014-01-30 23:00"),
*MustParseTime("2014-03-02 01:00"),
uint64(1),
); len(m) != 5 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestTimeIDsFromQuantum(t *testing.T) {
_ = pilosa.TimeIDsFromQuantum(pilosa.YMD, *MustParseTime("1970-01-01 00:00"), uint64(15027))
}
// DefaultTimeLayout is the time layout used by the tests.
const DefaultTimeLayout = "2006-01-02 15:04"
// MustParseTime parses value using DefaultTimeLayout. Panic on error.
func MustParseTime(value string) *time.Time {
v, err := time.Parse(DefaultTimeLayout, value)
if err != nil {
panic(err)
}
return &v
}

172
timeframe.go
View file

@ -1,172 +0,0 @@
package pilosa
import (
"time"
)
type TimeQuantum uint
const (
Y TimeQuantum = 3
YM TimeQuantum = 2
YMD TimeQuantum = 1
YMDH TimeQuantum = 0
)
func GetTimeID(t TimeQuantum, year uint, month uint, day uint, hour uint, tile_id uint64) uint64 {
y := year - 1970 //config.startyear
time_stamp := uint64((uint(t) << 30) | (y << 23) | (month << 19) | (day << 14) | (hour << 9))
time_stamp = time_stamp << 32
result := time_stamp | tile_id
return result
}
func GetTimeIds(tile_id uint64, atime time.Time, min_quantum TimeQuantum) []uint64 {
results := make([]uint64, 0, 4)
year := uint(atime.Year())
month := atime.Month()
day := atime.Day()
hour := atime.Hour()
if min_quantum <= Y {
id := GetTimeID(Y, year, 0, 0, 0, tile_id)
results = append(results, id)
}
if min_quantum <= YM {
id := GetTimeID(YM, uint(year), uint(month), 0, 0, tile_id)
results = append(results, id)
}
if min_quantum <= YMD {
id := GetTimeID(YMD, year, uint(month), uint(day), 0, tile_id)
results = append(results, id)
}
if min_quantum <= YMDH {
id := GetTimeID(YMDH, year, uint(month), uint(day), uint(hour), tile_id)
results = append(results, id)
}
return results
}
func IncrementYear(t time.Time) time.Time {
return t.AddDate(1, 0, 0)
}
func IncrementMonth(t time.Time) time.Time {
return t.AddDate(0, 1, 0)
}
func IncrementDay(t time.Time) time.Time {
return t.AddDate(0, 0, 1)
}
func IncrementHour(t time.Time) time.Time {
return t.Add(time.Hour)
}
func sameYear(t1, t2 time.Time) bool {
y1, _, _ := t1.Date()
y2, _, _ := t2.Date()
return (y1 == y2)
}
func NextYear(start time.Time, end time.Time) bool {
nextYear := start.AddDate(1, 0, 0)
return sameYear(nextYear, end) || end.After(nextYear)
}
func sameMonth(t1, t2 time.Time) bool {
y1, m1, _ := t1.Date()
y2, m2, _ := t2.Date()
return (y1 == y2) && (m1 == m2)
}
func NextMonth(start time.Time, end time.Time) bool {
nextMonth := start.AddDate(0, 1, 0)
return sameMonth(nextMonth, end) || end.After(nextMonth)
}
func sameDay(t1, t2 time.Time) bool {
y1, m1, d1 := t1.Date()
y2, m2, d2 := t2.Date()
return (y1 == y2) && (m1 == m2) && (d1 == d2)
}
func NextDay(start time.Time, end time.Time) bool {
nextDay := start.AddDate(0, 0, 1)
return sameDay(nextDay, end) || end.After(nextDay)
}
func GetRange(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 {
results, marker := upHill(start_time, end_time, tile_id)
r2 := downHill(marker, end_time, tile_id)
results = append(results, r2...)
return results
}
func upHill(start_time time.Time, end_time time.Time, tile_id uint64) ([]uint64, time.Time) {
var results []uint64
time_iterator := start_time
for time_iterator.Before(end_time) {
if NextDay(time_iterator, end_time) {
if time_iterator.Hour() == 0 {
if NextMonth(time_iterator, end_time) {
if time_iterator.Day() == 1 {
if NextYear(time_iterator, end_time) {
if time_iterator.Month() == 1 {
break
} else {
month_id := GetTimeID(YM, uint(time_iterator.Year()), uint(time_iterator.Month()), 0, 0, tile_id)
results = append(results, month_id)
time_iterator = IncrementMonth(time_iterator)
}
} else {
break
}
} else {
day_id := GetTimeID(YMD, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), 0, tile_id)
results = append(results, day_id)
time_iterator = IncrementDay(time_iterator)
}
} else {
break
}
} else {
hour_id := GetTimeID(YMDH, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), uint(time_iterator.Hour()), tile_id)
results = append(results, hour_id)
time_iterator = IncrementHour(time_iterator)
}
} else {
break
}
}
return results, time_iterator
}
func downHill(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 {
var results []uint64
time_iterator := start_time
for time_iterator.Before(end_time) {
if NextYear(time_iterator, end_time) {
year_id := GetTimeID(Y, uint(time_iterator.Year()), 0, 0, 0, tile_id)
results = append(results, year_id)
time_iterator = IncrementYear(time_iterator)
} else if NextMonth(time_iterator, end_time) {
month_id := GetTimeID(YM, uint(time_iterator.Year()), uint(time_iterator.Month()), 0, 0, tile_id)
results = append(results, month_id)
time_iterator = IncrementMonth(time_iterator)
} else if NextDay(time_iterator, end_time) {
day_id := GetTimeID(YMD, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), 0, tile_id)
results = append(results, day_id)
time_iterator = IncrementDay(time_iterator)
} else {
hour_id := GetTimeID(YMDH, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), uint(time_iterator.Hour()), tile_id)
results = append(results, hour_id)
time_iterator = IncrementHour(time_iterator)
}
}
return results
}

134
timeframe_test.go
View file

@ -1,134 +0,0 @@
package pilosa_test
import (
"testing"
"time"
"github.com/umbel/pilosa"
)
func TestGetRange_1h_0(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-08-11 14:00"),
MustParseTime("2014-08-11 16:00"),
uint64(1),
); len(m) != 2 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_1h_1(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-02 10:03"),
MustParseTime("2014-01-02 11:03"),
uint64(1),
); len(m) != 1 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_2h(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-02 10:03"),
MustParseTime("2014-01-02 12:03"),
uint64(1),
); len(m) != 2 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_24h(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-02 12:03"),
MustParseTime("2014-01-03 12:03"),
uint64(1),
); len(m) != 24 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_1d(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-02 00:00"),
MustParseTime("2014-01-03 00:00"),
uint64(1),
); len(m) != 1 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_1d1h(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-02 00:00"),
MustParseTime("2014-01-03 01:00"),
uint64(1),
); len(m) != 2 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_1h1d(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-02 23:00"),
MustParseTime("2014-01-04 00:00"),
uint64(1),
); len(m) != 2 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_1h1d1h(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-02 23:00"),
MustParseTime("2014-01-04 01:00"),
uint64(1),
); len(m) != 3 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_1y(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-01 00:00"),
MustParseTime("2015-01-01 00:00"),
uint64(1),
); len(m) != 1 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_1h1d1m(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-30 23:00"),
MustParseTime("2014-03-01 00:00"),
uint64(1),
); len(m) != 3 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetRange_1h1d1m1d1h(t *testing.T) {
if m := pilosa.GetRange(
MustParseTime("2014-01-30 23:00"),
MustParseTime("2014-03-02 01:00"),
uint64(1),
); len(m) != 5 {
t.Fatalf("unexpected range len: %d", len(m))
}
}
func TestGetTimeIds(t *testing.T) {
_ = pilosa.GetTimeIds(uint64(15027), MustParseTime("1970-01-01 00:00"), pilosa.YMD)
}
// DefaultTimeLayout is the time layout used by the tests.
const DefaultTimeLayout = "2006-01-02 15:04"
// MustParseTime parses value using DefaultTimeLayout. Panic on error.
func MustParseTime(value string) time.Time {
v, err := time.Parse(DefaultTimeLayout, value)
if err != nil {
panic(err)
}
return v
}