From cca897001987afa51f6a9cc91284fe7a85c1a907 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Mon, 22 Feb 2016 16:50:59 -0700 Subject: [PATCH] add Range() support This commit adds support for setting bits in time-based frames. --- attr.go | 6 ++ attr_test.go | 8 +- cmd/pilosa/main_test.go | 4 +- executor.go | 76 ++++++++++----- executor_test.go | 104 ++++++++++++-------- fragment.go | 33 +++++-- fragment_test.go | 31 +++--- frame.go | 8 ++ handler.go | 61 ++++++++++-- handler_test.go | 30 +++--- internal/internal.pb.go | 211 ++++++++++++++++------------------------ internal/internal.proto | 10 +- pql/parser.go | 2 +- pql/parser_test.go | 4 +- time.go | 157 ++++++++++++++++++++++++++++++ time_test.go | 134 +++++++++++++++++++++++++ timeframe.go | 172 -------------------------------- timeframe_test.go | 134 ------------------------- 18 files changed, 630 insertions(+), 555 deletions(-) create mode 100644 time.go create mode 100644 time_test.go delete mode 100644 timeframe.go delete mode 100644 timeframe_test.go diff --git a/attr.go b/attr.go index 0a2a0b5d2..38abd002e 100644 --- a/attr.go +++ b/attr.go @@ -112,6 +112,12 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { } switch v := v.(type) { + case int: + attr[k] = uint64(v) + case uint: + attr[k] = uint64(v) + case int64: + attr[k] = uint64(v) case string, uint64, bool: attr[k] = v default: diff --git a/attr_test.go b/attr_test.go index af421249c..c5a909642 100644 --- a/attr_test.go +++ b/attr_test.go @@ -15,9 +15,9 @@ func TestAttrStore_Attrs(t *testing.T) { defer s.Close() // Set attributes. - if err := s.SetAttrs(1, map[string]interface{}{"A": int64(100)}); err != nil { + if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil { t.Fatal(err) - } else if err := s.SetAttrs(2, map[string]interface{}{"A": int64(200)}); err != nil { + } else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil { t.Fatal(err) } else if err := s.SetAttrs(1, map[string]interface{}{"B": "VALUE"}); err != nil { t.Fatal(err) @@ -26,14 +26,14 @@ func TestAttrStore_Attrs(t *testing.T) { // Retrieve attributes for profile #1. if m, err := s.Attrs(1); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE"}) { + } else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(100), "B": "VALUE"}) { t.Fatalf("unexpected attrs(1): %#v", m) } // Retrieve attributes for profile #2. if m, err := s.Attrs(2); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) { + } else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(200)}) { t.Fatalf("unexpected attrs(2): %#v", m) } } diff --git a/cmd/pilosa/main_test.go b/cmd/pilosa/main_test.go index 9b99e3b72..6bdbcbef0 100644 --- a/cmd/pilosa/main_test.go +++ b/cmd/pilosa/main_test.go @@ -31,10 +31,8 @@ func TestMain_Set_Quick(t *testing.T) { // Execute SetBit() commands. for _, cmd := range cmds { - if res, err := m.Query("db=d", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil { + if _, err := m.Query("db=d", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil { t.Fatal(err) - } else if res != `{}`+"\n" { - t.Fatalf("unexpected result: %s", res) } } diff --git a/executor.go b/executor.go index 3e419c949..88cb2311c 100644 --- a/executor.go +++ b/executor.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "sort" + "time" "github.com/gogo/protobuf/proto" "github.com/umbel/pilosa/internal" @@ -41,16 +42,21 @@ func NewExecutor(index *Index) *Executor { func (e *Executor) Index() *Index { return e.index } // Execute executes a PQL query. -func (e *Executor) Execute(db string, q *pql.Query, slices []uint64) (interface{}, error) { +func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error) { // Verify that a database is set. if db == "" { return nil, ErrDatabaseRequired } + // Default options. + if opt == nil { + opt = &ExecOptions{} + } + // Ignore slices for set calls. switch root := q.Root.(type) { case *pql.SetBit: - return e.executeSetBit(db, root) + return e.executeSetBit(db, root, opt) case *pql.SetBitmapAttrs: return nil, e.executeSetBitmapAttrs(db, root) case *pql.SetProfileAttrs: @@ -70,27 +76,27 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64) (interface{ } } - return e.executeCall(db, q.Root, slices) + return e.executeCall(db, q.Root, slices, opt) } // executeCall executes a call. -func (e *Executor) executeCall(db string, c pql.Call, slices []uint64) (interface{}, error) { +func (e *Executor) executeCall(db string, c pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { switch c := c.(type) { case pql.BitmapCall: - return e.executeBitmapCall(db, c, slices) + return e.executeBitmapCall(db, c, slices, opt) case *pql.Count: - return e.executeCount(db, c, slices) + return e.executeCount(db, c, slices, opt) case *pql.Profile: - return e.executeProfile(db, c) + return e.executeProfile(db, c, opt) case *pql.TopN: - return e.executeTopN(db, c, slices) + return e.executeTopN(db, c, slices, opt) default: panic("unreachable") } } // executeBitmapCall executes a call that returns a bitmap. -func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint64) (*Bitmap, error) { +func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint64, opt *ExecOptions) (*Bitmap, error) { other := NewBitmap() for node, nodeSlices := range e.slicesByNode(slices) { // Execute locally if the hostname matches. @@ -106,7 +112,7 @@ func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint6 } // Otherwise execute remotely. - res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices) + res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt) if err != nil { return nil, err } @@ -147,7 +153,7 @@ func (e *Executor) executeBitmapCallSlice(db string, c pql.BitmapCall, slice uin } // executeTopN executes a TopN() call. -func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, error) { +func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64, opt *ExecOptions) ([]Pair, error) { var results []Pair for node, nodeSlices := range e.slicesByNode(slices) { // Execute locally if the hostname matches. @@ -163,7 +169,7 @@ func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, } // Otherwise execute remotely. - res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices) + res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt) if err != nil { return nil, err } @@ -260,7 +266,16 @@ func (e *Executor) executeIntersectSlice(db string, c *pql.Intersect, slice uint // executeRangeSlice executes a range() call for a local slice. func (e *Executor) executeRangeSlice(db string, c *pql.Range, slice uint64) (*Bitmap, error) { - panic("FIXME") + frame := c.Frame + if frame == "" { + frame = DefaultFrame + } + + f := e.Index().Fragment(db, frame, slice) + if f == nil { + return NewBitmap(), nil + } + return f.Range(c.ID, c.StartTime, c.EndTime), nil } // executeUnionSlice executes a union() call for a local slice. @@ -283,7 +298,7 @@ func (e *Executor) executeUnionSlice(db string, c *pql.Union, slice uint64) (*Bi } // executeCount executes a count() call. -func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint64, error) { +func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64, opt *ExecOptions) (uint64, error) { var n uint64 for node, nodeSlices := range e.slicesByNode(slices) { // Execute locally if the hostname matches. @@ -299,7 +314,7 @@ func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint6 } // Otherwise execute remotely. - res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices) + res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt) if err != nil { return 0, err } @@ -310,12 +325,12 @@ func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint6 // executeProfile executes a Profile() call. // This call only executes locally since the profile attibutes are stored locally. -func (e *Executor) executeProfile(db string, c *pql.Profile) (*Profile, error) { +func (e *Executor) executeProfile(db string, c *pql.Profile, opt *ExecOptions) (*Profile, error) { panic("FIXME: impl: e.Index().ProfileAttr(c.ID)") } // executeSetBit executes a SetBit() call. -func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) { +func (e *Executor) executeSetBit(db string, c *pql.SetBit, opt *ExecOptions) (bool, error) { slice := c.ProfileID / SliceWidth ret := false for _, node := range e.Cluster.SliceNodes(slice) { @@ -325,7 +340,7 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) { if err != nil { return false, fmt.Errorf("fragment: %s", err) } - val, err := f.SetBit(c.ID, c.ProfileID) + val, err := f.SetBit(c.ID, c.ProfileID, opt.Timestamp, opt.Quantum) if err != nil { return false, err } @@ -336,7 +351,7 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) { } // Forward call to remote node otherwise. - if _, err := e.exec(node, db, &pql.Query{Root: c}, nil); err != nil { + if _, err := e.exec(node, db, &pql.Query{Root: c}, nil, opt); err != nil { return false, err } fmt.Println("NEED TO IMPLEMENT REMOTE SETBIT") @@ -381,13 +396,18 @@ func (e *Executor) executeSetProfileAttrs(db string, c *pql.SetProfileAttrs) err } // exec executes a PQL query remotely for a set of slices on a node. -func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64) (result interface{}, err error) { +func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (result interface{}, err error) { // Encode request object. - buf, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String(db), - Query: proto.String(q.String()), - Slices: slices, - }) + pbreq := &internal.QueryRequest{ + DB: proto.String(db), + Query: proto.String(q.String()), + Slices: slices, + Quantum: proto.Uint32(uint32(opt.Quantum)), + } + if opt.Timestamp != nil { + pbreq.Timestamp = proto.Int64(opt.Timestamp.UnixNano()) + } + buf, err := proto.Marshal(pbreq) if err != nil { return nil, err } @@ -464,6 +484,12 @@ func (e *Executor) slicesByNode(slices []uint64) map[*Node][]uint64 { return m } +// ExecOptions represents an execution context for a single Execute() call. +type ExecOptions struct { + Timestamp *time.Time + Quantum TimeQuantum +} + // decodeError returns an error representation of s if s is non-blank. // Returns nil if s is blank. func decodeError(s string) error { diff --git a/executor_test.go b/executor_test.go index 703e16bc1..7a4d02613 100644 --- a/executor_test.go +++ b/executor_test.go @@ -17,12 +17,12 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3) idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1) - if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": 123}); err != nil { + if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -30,7 +30,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) } else if chunks[1].Value[0] != 2 { t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1])) - } else if attrs := res.(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": 123}) { + } else if attrs := res.(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } } @@ -45,7 +45,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 1 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -67,7 +67,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -90,7 +90,7 @@ func TestExecutor_Execute_Union(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -110,7 +110,7 @@ func TestExecutor_Execute_Count(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) - if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil { + if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if n != uint64(3) { t.Fatalf("unexpected n: %d", n) @@ -123,7 +123,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { defer idx.Close() e := NewExecutor(idx.Index, NewCluster(1)) - if _, err := e.Execute("d", MustParse(`SetBit(id=10, frame=f, profileID=1)`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBit(id=10, frame=f, profileID=1)`), nil, nil); err != nil { t.Fatal(err) } @@ -141,23 +141,23 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := NewExecutor(idx.Index, NewCluster(1)) - if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=XXX, YYY=1)`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=XXX, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } f := idx.Frame("d", "f") if m, err := f.BitmapAttrStore().Attrs(10); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { + } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": uint64(123), "bat": true}) { t.Fatalf("unexpected bitmap attr: %#v", m) } } @@ -168,19 +168,19 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0, nil, 0) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) - if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil); err != nil { + if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []pilosa.Pair{ {Key: 0, Count: 5}, @@ -196,23 +196,23 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2, nil, 0) // Create an intersecting bitmap. - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2, nil, 0) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) - if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil); err != nil { + if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []pilosa.Pair{ {Key: 20, Count: 3}, @@ -223,6 +223,24 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { } } +// Ensure a range query can be executed. +func TestExecutor_Execute_Range(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + f := idx.MustCreateFragmentIfNotExists("d", "f.t", 0) + if _, err := f.SetBit(1, 100, MustParseTime("2000-01-01 00:00"), pilosa.YMD); err != nil { + t.Fatal(err) + } + + e := NewExecutor(idx.Index, NewCluster(1)) + if res, err := e.Execute("d", MustParse(`Range(id=1, frame=f.t, start="2000-01-01T00:00", end="2000-01-01T01:00")`), nil, nil); err != nil { + t.Fatal(err) + } else if bits := res.(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) { + t.Fatalf("unexpected bits: %+v", bits) + } +} + // Ensure a remote query can return a bitmap. func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { c := NewCluster(2) @@ -233,7 +251,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Bitmap(id=10, frame=f)` { @@ -258,7 +276,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1) e := NewExecutor(idx.Index, c) - if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 3 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -279,7 +297,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to return a count. - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return uint64(10), nil } @@ -290,7 +308,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2) e := NewExecutor(idx.Index, c) - if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil { + if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if n != uint64(12) { t.Fatalf("unexpected n: %d", n) @@ -309,7 +327,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Mock secondary server's executor to verify arguments. var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) } else if query.String() != `SetBit(id=10, frame=f, profileID=2)` { @@ -324,7 +342,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { defer idx.Close() e := NewExecutor(idx.Index, c) - if _, err := e.Execute("d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -347,7 +365,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) } else if query.String() != `TopN(frame=f, n=3)` { @@ -371,7 +389,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 3).MustSetBits(30, (3*SliceWidth)+2) e := NewExecutor(idx.Index, c) - if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res, []pilosa.Pair{ {Key: 0, Count: 5}, diff --git a/fragment.go b/fragment.go index 176585ddf..6c5b1b7f6 100644 --- a/fragment.go +++ b/fragment.go @@ -188,7 +188,7 @@ func (f *Fragment) openStorage() error { // openCache initializes the cache from bitmap ids persisted to disk. func (f *Fragment) openCache() error { // Determine cache type from frame name. - if strings.HasSuffix(f.frame, ".n") { + if strings.HasSuffix(f.frame, FrameSuffixRank) { c := NewRankCache() c.ThresholdLength = 50000 c.ThresholdIndex = 45000 @@ -312,13 +312,19 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { // SetBit sets a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) { +func (f *Fragment) SetBit(bitmapID, profileID uint64, t *time.Time, q TimeQuantum) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() + + // Set time bits if this is a time-frame and a timestamp is specified. + if strings.HasSuffix(f.frame, FrameSuffixTime) && t != nil { + return f.setTimeBit(bitmapID, profileID, *t, q) + } + return f.setBit(bitmapID, profileID) } -func (f *Fragment) setBit(bitmapID, profileID uint64) (bool, error) { +func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) { // Determine the position of the bit in the storage. pos, err := f.pos(bitmapID, profileID) if err != nil { @@ -335,6 +341,17 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (bool, error) { } +func (f *Fragment) setTimeBit(bitmapID, profileID uint64, t time.Time, q TimeQuantum) (changed bool, err error) { + for _, timeID := range TimeIDsFromQuantum(q, t, bitmapID) { + if v, err := f.setBit(timeID, profileID); err != nil { + return changed, fmt.Errorf("set time bit: t=%s, q=%s, err=%s", t, q, err) + } else if v { + changed = true + } + } + return changed, nil +} + // ClearBit clears a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) { @@ -472,16 +489,18 @@ func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap { f.mu.Lock() defer f.mu.Unlock() - bitmapIDs := GetRange(start, end, bitmapID) + // Retrieve a list of bitmap ids for a given time range. + bitmapIDs := TimeIDsFromRange(start, end, bitmapID) if len(bitmapIDs) == 0 { return NewBitmap() } - result := f.bitmap(bitmapIDs[0]) + // Union all bitmap ids from the time range. + bm := f.bitmap(bitmapIDs[0]) for _, id := range bitmapIDs[1:] { - result = result.Union(f.bitmap(id)) + bm = bm.Union(f.bitmap(id)) } - return result + return bm } // Import bulk imports a set of bits and then snapshots the storage. diff --git a/fragment_test.go b/fragment_test.go index 441c254e5..e5903c124 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -19,11 +19,11 @@ func TestFragment_SetBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(120, 1); err != nil { + if _, err := f.SetBit(120, 1, nil, 0); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 6); err != nil { + } else if _, err := f.SetBit(120, 6, nil, 0); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(121, 0); err != nil { + } else if _, err := f.SetBit(121, 0, nil, 0); err != nil { t.Fatal(err) } @@ -50,9 +50,9 @@ func TestFragment_ClearBit(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.SetBit(1000, 1, nil, 0); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -77,9 +77,9 @@ func TestFragment_Snapshot(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.SetBit(1000, 1, nil, 0); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -133,11 +133,11 @@ func TestFragment_TopN_Filter(t *testing.T) { f.MustSetBits(102, 1, 2) // Assign attributes. - f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": 10}) - f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": 20}) + f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) + f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)}) // Retrieve top bitmaps. - if pairs, err := f.TopN(2, nil, "x", []interface{}{10, 15, 20}); err != nil { + if pairs, err := f.TopN(2, nil, "x", []interface{}{uint64(10), uint64(15), uint64(20)}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -176,6 +176,10 @@ func TestFragment_TopN_Intersect(t *testing.T) { // Ensure a fragment can return top bitmaps that have many bits set. func TestFragment_TopN_Intersect_Large(t *testing.T) { + if testing.Short() { + t.Skip("short mode") + } + f := MustOpenFragment("d", "f", 0) defer f.Close() @@ -218,7 +222,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.SetBit(i, 0, nil, 0); err != nil { t.Fatal(err) } } @@ -250,7 +254,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.SetBit(i, 0, nil, 0); err != nil { t.Fatal(err) } } @@ -330,9 +334,10 @@ func (f *Fragment) Reopen() error { } // MustSetBits sets bits on a bitmap. Panic on error. +// This function does not accept a timestamp or quantum. func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { for _, profileID := range profileIDs { - if _, err := f.SetBit(bitmapID, profileID); err != nil { + if _, err := f.SetBit(bitmapID, profileID, nil, 0); err != nil { panic(err) } } diff --git a/frame.go b/frame.go index 6142f5ec7..9dd48d615 100644 --- a/frame.go +++ b/frame.go @@ -8,6 +8,14 @@ import ( "sync" ) +const ( + // FrameSuffixTime is the suffix used for time-based frames. + FrameSuffixTime = ".t" + + // FrameSuffixRank is the suffix used for rank-based frames. + FrameSuffixRank = ".n" +) + // Frame represents a container for fragments. type Frame struct { mu sync.Mutex diff --git a/handler.go b/handler.go index c6927a14e..52940e7e2 100644 --- a/handler.go +++ b/handler.go @@ -29,7 +29,7 @@ type Handler struct { // The execution engine for running queries. Executor interface { - Execute(db string, query *pql.Query, slices []uint64) (interface{}, error) + Execute(db string, query *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error) } // The version to report on the /version endpoint. @@ -95,6 +95,12 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { return } + // Build execution options. + opt := &ExecOptions{ + Timestamp: req.Timestamp, + Quantum: req.Quantum, + } + // Parse query string. q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { @@ -104,7 +110,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } // Execute the query. - result, err := h.Executor.Execute(req.DB, q, req.Slices) + result, err := h.Executor.Execute(req.DB, q, req.Slices, opt) resp := &QueryResponse{Result: result, Err: err} // Fill profile attributes if requested. @@ -196,11 +202,38 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { return nil, errors.New("invalid slice argument") } + // Parse timestamp, if available. + var timestamp *time.Time + if v := q.Get("timestamp"); v != "" { + layout := "2006-01-02 15:04:05" + if strings.Contains(v, "T") { + layout = "2006-01-02T15:04:05" + } + + t, err := time.Parse(layout, v) + if err != nil { + return nil, errors.New("invalid timestamp") + } + timestamp = &t + } + + // Parse time granularity. + quantum := YMDH + if s := q.Get("time_granularity"); s != "" { + v, err := ParseTimeQuantum(s) + if err != nil { + return nil, errors.New("invalid time granularity") + } + quantum = v + } + return &QueryRequest{ - DB: q.Get("db"), - Query: query, - Slices: slices, - Profiles: q.Get("profiles") == "true", + DB: q.Get("db"), + Query: query, + Slices: slices, + Profiles: q.Get("profiles") == "true", + Timestamp: timestamp, + Quantum: quantum, }, nil } @@ -352,15 +385,29 @@ type QueryRequest struct { // Return profile attributes, if true. Profiles bool + + // Timestamp passed into the query. + Timestamp *time.Time + + // Time granularity to use with the timestamp. + Quantum TimeQuantum } func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { - return &QueryRequest{ + req := &QueryRequest{ DB: pb.GetDB(), Query: pb.GetQuery(), Slices: pb.GetSlices(), Profiles: pb.GetProfiles(), + Quantum: TimeQuantum(pb.GetQuantum()), } + + if pb.Timestamp != nil { + t := time.Unix(0, pb.GetTimestamp()) + req.Timestamp = &t + } + + return req } // QueryResponse represent a response from a processed query. diff --git a/handler_test.go b/handler_test.go index a8c3a64f8..65d000c3e 100644 --- a/handler_test.go +++ b/handler_test.go @@ -29,7 +29,7 @@ func TestHandler_NotFound(t *testing.T) { // Ensure the handler can accept URL arguments. func TestHandler_Query_Args_URL(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != "db0" { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Count(Bitmap(id=100))` { @@ -52,7 +52,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { // Ensure the handler can accept arguments via protobufs. func TestHandler_Query_Args_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != "db0" { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Count(Bitmap(id=100))` { @@ -98,7 +98,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as JSON. func TestHandler_Query_Uint64_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return uint64(100), nil } @@ -114,7 +114,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as protobufs. func TestHandler_Query_Uint64_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return uint64(100), nil } @@ -137,7 +137,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as JSON. func TestHandler_Query_Bitmap_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} return bm, nil @@ -169,7 +169,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { h := NewHandler() h.Index = idx.Index - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} return bm, nil @@ -187,7 +187,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as protobuf. func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} return bm, nil @@ -232,7 +232,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { h := NewHandler() h.Index = idx.Index - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} return bm, nil @@ -287,7 +287,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns pairs as JSON. func TestHandler_Query_Pairs_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return []pilosa.Pair{ {Key: 1, Count: 2}, {Key: 3, Count: 4}, @@ -306,7 +306,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns pairs as protobuf. func TestHandler_Query_Pairs_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return []pilosa.Pair{ {Key: 1, Count: 2}, {Key: 3, Count: 4}, @@ -332,7 +332,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { // Ensure the handler can return an error as JSON. func TestHandler_Query_Err_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return nil, errors.New("marker") } @@ -348,7 +348,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { // Ensure the handler can return an error as protobuf. func TestHandler_Query_Err_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return nil, errors.New("marker") } @@ -449,13 +449,13 @@ func NewHandler() *Handler { // HandlerExecutor is a mock implementing pilosa.Handler.Executor. type HandlerExecutor struct { cluster *pilosa.Cluster - ExecuteFn func(db string, query *pql.Query, slices []uint64) (interface{}, error) + ExecuteFn func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) } func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } -func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64) (interface{}, error) { - return c.ExecuteFn(db, query, slices) +func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + return c.ExecuteFn(db, query, slices, opt) } // Server represents a test wrapper for httptest.Server. diff --git a/internal/internal.pb.go b/internal/internal.pb.go index ef49d0d53..9ca780d28 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -1,12 +1,12 @@ -// Code generated by protoc-gen-go. -// source: internal.proto +// Code generated by protoc-gen-gogo. +// source: internal/internal.proto // DO NOT EDIT! /* Package internal is a generated protocol buffer package. It is generated from these files: - internal.proto + internal/internal.proto It has these top-level messages: Bitmap @@ -25,24 +25,21 @@ It has these top-level messages: package internal import proto "github.com/gogo/protobuf/proto" -import fmt "fmt" import math "math" // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal -var _ = fmt.Errorf var _ = math.Inf type Bitmap struct { - Chunks []*Chunk `protobuf:"bytes,1,rep,name=Chunks" json:"Chunks,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` + Chunks []*Chunk `protobuf:"bytes,1,rep" json:"Chunks,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep" json:"Attrs,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *Bitmap) Reset() { *m = Bitmap{} } -func (m *Bitmap) String() string { return proto.CompactTextString(m) } -func (*Bitmap) ProtoMessage() {} -func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } +func (m *Bitmap) Reset() { *m = Bitmap{} } +func (m *Bitmap) String() string { return proto.CompactTextString(m) } +func (*Bitmap) ProtoMessage() {} func (m *Bitmap) GetChunks() []*Chunk { if m != nil { @@ -59,15 +56,14 @@ func (m *Bitmap) GetAttrs() []*Attr { } type Chunk struct { - Key *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"` - Value []uint64 `protobuf:"varint,2,rep,name=Value" json:"Value,omitempty"` + Key *uint64 `protobuf:"varint,1,req" json:"Key,omitempty"` + Value []uint64 `protobuf:"varint,2,rep" json:"Value,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *Chunk) Reset() { *m = Chunk{} } -func (m *Chunk) String() string { return proto.CompactTextString(m) } -func (*Chunk) ProtoMessage() {} -func (*Chunk) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } +func (m *Chunk) Reset() { *m = Chunk{} } +func (m *Chunk) String() string { return proto.CompactTextString(m) } +func (*Chunk) ProtoMessage() {} func (m *Chunk) GetKey() uint64 { if m != nil && m.Key != nil { @@ -84,15 +80,14 @@ func (m *Chunk) GetValue() []uint64 { } type Pair struct { - Key *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"` - Count *uint64 `protobuf:"varint,2,req,name=Count" json:"Count,omitempty"` + Key *uint64 `protobuf:"varint,1,req" json:"Key,omitempty"` + Count *uint64 `protobuf:"varint,2,req" json:"Count,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *Pair) Reset() { *m = Pair{} } -func (m *Pair) String() string { return proto.CompactTextString(m) } -func (*Pair) ProtoMessage() {} -func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } +func (m *Pair) Reset() { *m = Pair{} } +func (m *Pair) String() string { return proto.CompactTextString(m) } +func (*Pair) ProtoMessage() {} func (m *Pair) GetKey() uint64 { if m != nil && m.Key != nil { @@ -109,15 +104,14 @@ func (m *Pair) GetCount() uint64 { } type Bit struct { - BitmapID *uint64 `protobuf:"varint,1,req,name=BitmapID" json:"BitmapID,omitempty"` - ProfileID *uint64 `protobuf:"varint,2,req,name=ProfileID" json:"ProfileID,omitempty"` + BitmapID *uint64 `protobuf:"varint,1,req" json:"BitmapID,omitempty"` + ProfileID *uint64 `protobuf:"varint,2,req" json:"ProfileID,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *Bit) Reset() { *m = Bit{} } -func (m *Bit) String() string { return proto.CompactTextString(m) } -func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } +func (m *Bit) Reset() { *m = Bit{} } +func (m *Bit) String() string { return proto.CompactTextString(m) } +func (*Bit) ProtoMessage() {} func (m *Bit) GetBitmapID() uint64 { if m != nil && m.BitmapID != nil { @@ -134,15 +128,14 @@ func (m *Bit) GetProfileID() uint64 { } type Profile struct { - ID *uint64 `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` + ID *uint64 `protobuf:"varint,1,req" json:"ID,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep" json:"Attrs,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *Profile) Reset() { *m = Profile{} } -func (m *Profile) String() string { return proto.CompactTextString(m) } -func (*Profile) ProtoMessage() {} -func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } +func (m *Profile) Reset() { *m = Profile{} } +func (m *Profile) String() string { return proto.CompactTextString(m) } +func (*Profile) ProtoMessage() {} func (m *Profile) GetID() uint64 { if m != nil && m.ID != nil { @@ -159,17 +152,16 @@ func (m *Profile) GetAttrs() []*Attr { } type Attr struct { - Key *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"` - StringValue *string `protobuf:"bytes,2,opt,name=StringValue" json:"StringValue,omitempty"` - UintValue *uint64 `protobuf:"varint,3,opt,name=UintValue" json:"UintValue,omitempty"` - BoolValue *bool `protobuf:"varint,4,opt,name=BoolValue" json:"BoolValue,omitempty"` + Key *string `protobuf:"bytes,1,req" json:"Key,omitempty"` + StringValue *string `protobuf:"bytes,2,opt" json:"StringValue,omitempty"` + UintValue *uint64 `protobuf:"varint,3,opt" json:"UintValue,omitempty"` + BoolValue *bool `protobuf:"varint,4,opt" json:"BoolValue,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *Attr) Reset() { *m = Attr{} } -func (m *Attr) String() string { return proto.CompactTextString(m) } -func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +func (m *Attr) Reset() { *m = Attr{} } +func (m *Attr) String() string { return proto.CompactTextString(m) } +func (*Attr) ProtoMessage() {} func (m *Attr) GetKey() string { if m != nil && m.Key != nil { @@ -200,14 +192,13 @@ func (m *Attr) GetBoolValue() bool { } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` + Attrs []*Attr `protobuf:"bytes,1,rep" json:"Attrs,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *AttrMap) Reset() { *m = AttrMap{} } -func (m *AttrMap) String() string { return proto.CompactTextString(m) } -func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (m *AttrMap) Reset() { *m = AttrMap{} } +func (m *AttrMap) String() string { return proto.CompactTextString(m) } +func (*AttrMap) ProtoMessage() {} func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -217,17 +208,18 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Query *string `protobuf:"bytes,2,req,name=Query" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,3,rep,name=Slices" json:"Slices,omitempty"` - Profiles *bool `protobuf:"varint,4,opt,name=Profiles" json:"Profiles,omitempty"` + DB *string `protobuf:"bytes,1,req" json:"DB,omitempty"` + Query *string `protobuf:"bytes,2,req" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,3,rep" json:"Slices,omitempty"` + Profiles *bool `protobuf:"varint,4,opt" json:"Profiles,omitempty"` + Timestamp *int64 `protobuf:"varint,5,opt" json:"Timestamp,omitempty"` + Quantum *uint32 `protobuf:"varint,6,opt" json:"Quantum,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *QueryRequest) Reset() { *m = QueryRequest{} } -func (m *QueryRequest) String() string { return proto.CompactTextString(m) } -func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } +func (m *QueryRequest) Reset() { *m = QueryRequest{} } +func (m *QueryRequest) String() string { return proto.CompactTextString(m) } +func (*QueryRequest) ProtoMessage() {} func (m *QueryRequest) GetDB() string { if m != nil && m.DB != nil { @@ -257,19 +249,32 @@ func (m *QueryRequest) GetProfiles() bool { return false } +func (m *QueryRequest) GetTimestamp() int64 { + if m != nil && m.Timestamp != nil { + return *m.Timestamp + } + return 0 +} + +func (m *QueryRequest) GetQuantum() uint32 { + if m != nil && m.Quantum != nil { + return *m.Quantum + } + return 0 +} + type QueryResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"` - Bitmap *Bitmap `protobuf:"bytes,2,opt,name=Bitmap" json:"Bitmap,omitempty"` - N *uint64 `protobuf:"varint,3,opt,name=N" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,4,rep,name=Pairs" json:"Pairs,omitempty"` - Profiles []*Profile `protobuf:"bytes,5,rep,name=Profiles" json:"Profiles,omitempty"` + Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"` + Bitmap *Bitmap `protobuf:"bytes,2,opt" json:"Bitmap,omitempty"` + N *uint64 `protobuf:"varint,3,opt" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,4,rep" json:"Pairs,omitempty"` + Profiles []*Profile `protobuf:"bytes,5,rep" json:"Profiles,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *QueryResponse) Reset() { *m = QueryResponse{} } -func (m *QueryResponse) String() string { return proto.CompactTextString(m) } -func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } +func (m *QueryResponse) Reset() { *m = QueryResponse{} } +func (m *QueryResponse) String() string { return proto.CompactTextString(m) } +func (*QueryResponse) ProtoMessage() {} func (m *QueryResponse) GetErr() string { if m != nil && m.Err != nil { @@ -307,18 +312,17 @@ func (m *QueryResponse) GetProfiles() []*Profile { } type ImportRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Frame *string `protobuf:"bytes,2,req,name=Frame" json:"Frame,omitempty"` - Slice *uint64 `protobuf:"varint,3,req,name=Slice" json:"Slice,omitempty"` - BitmapIDs []uint64 `protobuf:"varint,4,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,5,rep,name=ProfileIDs" json:"ProfileIDs,omitempty"` + DB *string `protobuf:"bytes,1,req" json:"DB,omitempty"` + Frame *string `protobuf:"bytes,2,req" json:"Frame,omitempty"` + Slice *uint64 `protobuf:"varint,3,req" json:"Slice,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,4,rep" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,5,rep" json:"ProfileIDs,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *ImportRequest) Reset() { *m = ImportRequest{} } -func (m *ImportRequest) String() string { return proto.CompactTextString(m) } -func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } +func (m *ImportRequest) Reset() { *m = ImportRequest{} } +func (m *ImportRequest) String() string { return proto.CompactTextString(m) } +func (*ImportRequest) ProtoMessage() {} func (m *ImportRequest) GetDB() string { if m != nil && m.DB != nil { @@ -356,14 +360,13 @@ func (m *ImportRequest) GetProfileIDs() []uint64 { } type ImportResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"` + Err *string `protobuf:"bytes,1,opt" json:"Err,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *ImportResponse) Reset() { *m = ImportResponse{} } -func (m *ImportResponse) String() string { return proto.CompactTextString(m) } -func (*ImportResponse) ProtoMessage() {} -func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } +func (m *ImportResponse) Reset() { *m = ImportResponse{} } +func (m *ImportResponse) String() string { return proto.CompactTextString(m) } +func (*ImportResponse) ProtoMessage() {} func (m *ImportResponse) GetErr() string { if m != nil && m.Err != nil { @@ -373,14 +376,13 @@ func (m *ImportResponse) GetErr() string { } type Cache struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,1,rep" json:"BitmapIDs,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *Cache) Reset() { *m = Cache{} } -func (m *Cache) String() string { return proto.CompactTextString(m) } -func (*Cache) ProtoMessage() {} -func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } +func (m *Cache) Reset() { *m = Cache{} } +func (m *Cache) String() string { return proto.CompactTextString(m) } +func (*Cache) ProtoMessage() {} func (m *Cache) GetBitmapIDs() []uint64 { if m != nil { @@ -390,45 +392,4 @@ func (m *Cache) GetBitmapIDs() []uint64 { } func init() { - proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") - proto.RegisterType((*Chunk)(nil), "internal.Chunk") - proto.RegisterType((*Pair)(nil), "internal.Pair") - proto.RegisterType((*Bit)(nil), "internal.Bit") - proto.RegisterType((*Profile)(nil), "internal.Profile") - proto.RegisterType((*Attr)(nil), "internal.Attr") - proto.RegisterType((*AttrMap)(nil), "internal.AttrMap") - proto.RegisterType((*QueryRequest)(nil), "internal.QueryRequest") - proto.RegisterType((*QueryResponse)(nil), "internal.QueryResponse") - proto.RegisterType((*ImportRequest)(nil), "internal.ImportRequest") - proto.RegisterType((*ImportResponse)(nil), "internal.ImportResponse") - proto.RegisterType((*Cache)(nil), "internal.Cache") -} - -var fileDescriptor0 = []byte{ - // 398 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x84, 0x52, 0x4d, 0x8f, 0xda, 0x30, - 0x14, 0x54, 0x88, 0x03, 0xe4, 0xa5, 0xa4, 0xe0, 0x5e, 0x50, 0x25, 0x54, 0x64, 0x2e, 0xa8, 0x07, - 0x0e, 0xa8, 0x7f, 0xa0, 0x40, 0xab, 0x22, 0x54, 0x44, 0x8b, 0xda, 0x73, 0x23, 0xe4, 0x96, 0xa8, - 0x21, 0xce, 0x3a, 0xce, 0x81, 0x1f, 0xb1, 0xff, 0x79, 0x9f, 0x3f, 0x12, 0xd8, 0x5d, 0x56, 0x7b, - 0x8a, 0x3c, 0x1e, 0xbf, 0x99, 0x79, 0x13, 0x88, 0xd3, 0x5c, 0x71, 0x99, 0x27, 0xd9, 0xac, 0x90, - 0x42, 0x09, 0xda, 0xad, 0xcf, 0xec, 0x1b, 0xb4, 0x17, 0xa9, 0x3a, 0x25, 0x05, 0xfd, 0x00, 0xed, - 0xe5, 0xb1, 0xca, 0xff, 0x97, 0x43, 0x6f, 0xec, 0x4f, 0xa3, 0xf9, 0xdb, 0x59, 0xf3, 0xc8, 0xe0, - 0x74, 0x04, 0xc1, 0x67, 0xa5, 0x64, 0x39, 0x6c, 0x99, 0xfb, 0xf8, 0x72, 0xaf, 0x61, 0x36, 0x81, - 0xc0, 0xf2, 0x22, 0xf0, 0x37, 0xfc, 0x8c, 0x53, 0x5a, 0x53, 0x42, 0x7b, 0x10, 0xfc, 0x4e, 0xb2, - 0x8a, 0x9b, 0x47, 0x84, 0x31, 0x20, 0xbb, 0x24, 0x95, 0xcf, 0x38, 0x4b, 0x51, 0xe5, 0x0a, 0x39, - 0x78, 0x64, 0x1f, 0xc1, 0x47, 0x4b, 0xb4, 0x0f, 0x5d, 0xeb, 0x6c, 0xbd, 0x72, 0xbc, 0x01, 0x84, - 0x3b, 0x29, 0xfe, 0xa6, 0x19, 0x47, 0xc8, 0x72, 0x3f, 0x41, 0xc7, 0x41, 0x14, 0xa0, 0xd5, 0x30, - 0x5f, 0xb1, 0xba, 0x05, 0xa2, 0xbf, 0xd7, 0x2e, 0x42, 0xfa, 0x0e, 0xa2, 0xbd, 0x92, 0x69, 0xfe, - 0xaf, 0xf6, 0xeb, 0x21, 0x88, 0x92, 0xbf, 0xf0, 0xad, 0x85, 0x7c, 0x84, 0x8c, 0x8b, 0x85, 0x10, - 0x99, 0x85, 0x08, 0x42, 0x5d, 0x36, 0x85, 0x8e, 0x9e, 0xf7, 0x1d, 0xb7, 0xd8, 0x28, 0x7b, 0x37, - 0x95, 0x37, 0xf0, 0xe6, 0x47, 0xc5, 0xe5, 0xf9, 0x27, 0xbf, 0xab, 0x78, 0xa9, 0xb4, 0xe9, 0xd5, - 0xc2, 0x19, 0xc0, 0x35, 0x98, 0x3b, 0x13, 0x2d, 0xa4, 0x31, 0xb4, 0xf7, 0x59, 0x7a, 0xe0, 0x25, - 0xea, 0xe2, 0xea, 0xf4, 0x3e, 0x5c, 0xd4, 0xd2, 0xc9, 0xde, 0x7b, 0xd0, 0x73, 0xd3, 0xca, 0x42, - 0xe4, 0x25, 0xd7, 0x81, 0xbe, 0x48, 0x89, 0xf3, 0xb4, 0xf7, 0x71, 0x5d, 0xad, 0xc9, 0x12, 0xcd, - 0xfb, 0x17, 0x2f, 0xae, 0xf2, 0x10, 0xbc, 0xad, 0x4b, 0x85, 0xbe, 0x75, 0x31, 0x7a, 0xf4, 0x13, - 0xdf, 0xa6, 0xaf, 0xc9, 0x95, 0x78, 0x60, 0x18, 0x83, 0x2b, 0x86, 0xbd, 0x61, 0x7f, 0xa0, 0xb7, - 0x3e, 0x15, 0x42, 0xaa, 0x17, 0xd2, 0x7d, 0x95, 0xc9, 0x89, 0xbb, 0x74, 0x78, 0x34, 0xe9, 0x50, - 0xde, 0x55, 0x5b, 0x97, 0x6d, 0x2d, 0x10, 0x8a, 0xaf, 0x9b, 0xb6, 0xad, 0x28, 0x61, 0x23, 0x88, - 0x6b, 0x85, 0x1b, 0x89, 0xd9, 0x7b, 0xfc, 0x91, 0x92, 0xc3, 0x91, 0x3f, 0x1e, 0xa7, 0x9b, 0x20, - 0x0f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x4e, 0xbb, 0x74, 0xfe, 0x03, 0x03, 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index 3fe5b6b83..81dd0d564 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -37,10 +37,12 @@ message AttrMap { } message QueryRequest { - required string DB = 1; - required string Query = 2; - repeated uint64 Slices = 3; - optional bool Profiles = 4; + required string DB = 1; + required string Query = 2; + repeated uint64 Slices = 3; + optional bool Profiles = 4; + optional int64 Timestamp = 5; + optional uint32 Quantum = 6; } message QueryResponse { diff --git a/pql/parser.go b/pql/parser.go index 5f7025121..4761aeb88 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -441,7 +441,7 @@ func (p *Parser) parseSetProfileAttrsCall() (*SetProfileAttrs, error) { case string, bool: c.Attrs[key] = v case uint64: - c.Attrs[key] = int64(v) + c.Attrs[key] = v default: return nil, parseErrorf(pos, "invalid SetProfileAttrs() arg: %v", arg.key) } diff --git a/pql/parser_test.go b/pql/parser_test.go index ad4e3a1aa..df4bcf810 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -222,7 +222,7 @@ func TestParser_Parse_SetBitmapAttrs_Key(t *testing.T) { Frame: "b.n", Attrs: map[string]interface{}{ "foo": "bar", - "bar": int64(123), + "bar": uint64(123), "baz": true, "bat": false, "x": nil, @@ -244,7 +244,7 @@ func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) { Frame: "b.n", Attrs: map[string]interface{}{ "foo": "bar", - "bar": int64(123), + "bar": uint64(123), }, }, }) { diff --git a/time.go b/time.go new file mode 100644 index 000000000..ce11ea1d9 --- /dev/null +++ b/time.go @@ -0,0 +1,157 @@ +package pilosa + +import ( + "errors" + "strings" + "time" +) + +// TimeQuantum represents a time granularity for time-based bitmap ids. +type TimeQuantum uint + +// ParseTimeQuantum parses s into a quantum. +func ParseTimeQuantum(s string) (TimeQuantum, error) { + switch strings.ToUpper(s) { + case "Y": + return Y, nil + case "M": + return YM, nil + case "D": + return YMD, nil + case "H": + return YMDH, nil + default: + return 0, errors.New("invalid quantum") + } +} + +// String returns the string representation of the quantum. +func (q TimeQuantum) String() string { + switch q { + case Y: + return "Y" + case YM: + return "M" + case YMD: + return "D" + case YMDH: + return "H" + default: + return "Y" + } +} + +const ( + Y TimeQuantum = 3 + YM TimeQuantum = 2 + YMD TimeQuantum = 1 + YMDH TimeQuantum = 0 +) + +// TimeID returns a packed time identifier from a year/month/day/hour & tile. +func TimeID(q TimeQuantum, year uint, month uint, day uint, hour uint, tileID uint64) uint64 { + v := uint64((uint(q) << 30) | ((year - 1970) << 23) | (month << 19) | (day << 14) | (hour << 9)) + return (v << 32) | tileID +} + +// TimeIDsFromRange returns timestamp bitmap ids within a time range. +func TimeIDsFromRange(start, end time.Time, tileID uint64) []uint64 { + t := start + + var results []uint64 + for t.Before(end) { + if !nextDay(t, end) { + break + } + if t.Hour() == 0 { + if !nextMonth(t, end) { + break + } + + if t.Day() == 1 { + if !nextYear(t, end) { + break + } + + if t.Month() == 1 { + break + } + + results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) + t = t.AddDate(0, 1, 0) + } else { + results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) + t = t.AddDate(0, 0, 1) + } + } else { + results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) + t = t.Add(time.Hour) + } + } + + for t.Before(end) { + if nextYear(t, end) { + results = append(results, TimeID(Y, uint(t.Year()), 0, 0, 0, tileID)) + t = t.AddDate(1, 0, 0) + } else if nextMonth(t, end) { + results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) + t = t.AddDate(0, 1, 0) + } else if nextDay(t, end) { + results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) + t = t.AddDate(0, 0, 1) + } else { + results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) + t = t.Add(time.Hour) + } + } + + return results +} + +func nextYear(start time.Time, end time.Time) bool { + next := start.AddDate(1, 0, 0) + if next.Year() == end.Year() { + return true + } + return end.After(next) +} + +func nextMonth(start time.Time, end time.Time) bool { + next := start.AddDate(0, 1, 0) + y1, m1, _ := next.Date() + y2, m2, _ := end.Date() + if (y1 == y2) && (m1 == m2) { + return true + } + return end.After(next) +} + +func nextDay(start time.Time, end time.Time) bool { + next := start.AddDate(0, 0, 1) + y1, m1, d1 := next.Date() + y2, m2, d2 := end.Date() + if (y1 == y2) && (m1 == m2) && (d1 == d2) { + return true + } + return end.After(next) +} + +// TimeIDsFromQuantum returns a list of time identifiers for a single quantum. +func TimeIDsFromQuantum(q TimeQuantum, t time.Time, tileID uint64) []uint64 { + y, m, d, h := uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()) + + v := make([]uint64, 0, 4) + if q <= Y { + v = append(v, TimeID(Y, y, 0, 0, 0, tileID)) + } + if q <= YM { + v = append(v, TimeID(YM, y, m, 0, 0, tileID)) + } + if q <= YMD { + v = append(v, TimeID(YMD, y, m, d, 0, tileID)) + } + if q <= YMDH { + v = append(v, TimeID(YMDH, y, m, d, h, tileID)) + } + return v +} diff --git a/time_test.go b/time_test.go new file mode 100644 index 000000000..2d7644841 --- /dev/null +++ b/time_test.go @@ -0,0 +1,134 @@ +package pilosa_test + +import ( + "testing" + "time" + + "github.com/umbel/pilosa" +) + +func TestTimeIDsFromRange_1h_0(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-08-11 14:00"), + *MustParseTime("2014-08-11 16:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h_1(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 10:03"), + *MustParseTime("2014-01-02 11:03"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_2h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 10:03"), + *MustParseTime("2014-01-02 12:03"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_24h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 12:03"), + *MustParseTime("2014-01-03 12:03"), + uint64(1), + ); len(m) != 24 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1d(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 00:00"), + *MustParseTime("2014-01-03 00:00"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1d1h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 00:00"), + *MustParseTime("2014-01-03 01:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h1d(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 23:00"), + *MustParseTime("2014-01-04 00:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h1d1h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 23:00"), + *MustParseTime("2014-01-04 01:00"), + uint64(1), + ); len(m) != 3 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1y(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-01 00:00"), + *MustParseTime("2015-01-01 00:00"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h1d1m(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-30 23:00"), + *MustParseTime("2014-03-01 00:00"), + uint64(1), + ); len(m) != 3 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h1d1m1d1h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-30 23:00"), + *MustParseTime("2014-03-02 01:00"), + uint64(1), + ); len(m) != 5 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromQuantum(t *testing.T) { + _ = pilosa.TimeIDsFromQuantum(pilosa.YMD, *MustParseTime("1970-01-01 00:00"), uint64(15027)) +} + +// DefaultTimeLayout is the time layout used by the tests. +const DefaultTimeLayout = "2006-01-02 15:04" + +// MustParseTime parses value using DefaultTimeLayout. Panic on error. +func MustParseTime(value string) *time.Time { + v, err := time.Parse(DefaultTimeLayout, value) + if err != nil { + panic(err) + } + return &v +} diff --git a/timeframe.go b/timeframe.go deleted file mode 100644 index 22c75cc11..000000000 --- a/timeframe.go +++ /dev/null @@ -1,172 +0,0 @@ -package pilosa - -import ( - "time" -) - -type TimeQuantum uint - -const ( - Y TimeQuantum = 3 - YM TimeQuantum = 2 - YMD TimeQuantum = 1 - YMDH TimeQuantum = 0 -) - -func GetTimeID(t TimeQuantum, year uint, month uint, day uint, hour uint, tile_id uint64) uint64 { - y := year - 1970 //config.startyear - time_stamp := uint64((uint(t) << 30) | (y << 23) | (month << 19) | (day << 14) | (hour << 9)) - time_stamp = time_stamp << 32 - result := time_stamp | tile_id - - return result -} - -func GetTimeIds(tile_id uint64, atime time.Time, min_quantum TimeQuantum) []uint64 { - results := make([]uint64, 0, 4) - year := uint(atime.Year()) - month := atime.Month() - day := atime.Day() - hour := atime.Hour() - - if min_quantum <= Y { - id := GetTimeID(Y, year, 0, 0, 0, tile_id) - results = append(results, id) - } - - if min_quantum <= YM { - id := GetTimeID(YM, uint(year), uint(month), 0, 0, tile_id) - results = append(results, id) - } - - if min_quantum <= YMD { - id := GetTimeID(YMD, year, uint(month), uint(day), 0, tile_id) - results = append(results, id) - } - - if min_quantum <= YMDH { - id := GetTimeID(YMDH, year, uint(month), uint(day), uint(hour), tile_id) - results = append(results, id) - } - - return results -} - -func IncrementYear(t time.Time) time.Time { - return t.AddDate(1, 0, 0) -} -func IncrementMonth(t time.Time) time.Time { - return t.AddDate(0, 1, 0) -} -func IncrementDay(t time.Time) time.Time { - return t.AddDate(0, 0, 1) -} -func IncrementHour(t time.Time) time.Time { - return t.Add(time.Hour) -} - -func sameYear(t1, t2 time.Time) bool { - y1, _, _ := t1.Date() - y2, _, _ := t2.Date() - return (y1 == y2) -} - -func NextYear(start time.Time, end time.Time) bool { - nextYear := start.AddDate(1, 0, 0) - return sameYear(nextYear, end) || end.After(nextYear) -} - -func sameMonth(t1, t2 time.Time) bool { - y1, m1, _ := t1.Date() - y2, m2, _ := t2.Date() - return (y1 == y2) && (m1 == m2) -} - -func NextMonth(start time.Time, end time.Time) bool { - nextMonth := start.AddDate(0, 1, 0) - return sameMonth(nextMonth, end) || end.After(nextMonth) -} - -func sameDay(t1, t2 time.Time) bool { - y1, m1, d1 := t1.Date() - y2, m2, d2 := t2.Date() - return (y1 == y2) && (m1 == m2) && (d1 == d2) -} - -func NextDay(start time.Time, end time.Time) bool { - nextDay := start.AddDate(0, 0, 1) - return sameDay(nextDay, end) || end.After(nextDay) -} - -func GetRange(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 { - results, marker := upHill(start_time, end_time, tile_id) - r2 := downHill(marker, end_time, tile_id) - results = append(results, r2...) - return results -} - -func upHill(start_time time.Time, end_time time.Time, tile_id uint64) ([]uint64, time.Time) { - var results []uint64 - time_iterator := start_time - for time_iterator.Before(end_time) { - if NextDay(time_iterator, end_time) { - if time_iterator.Hour() == 0 { - if NextMonth(time_iterator, end_time) { - if time_iterator.Day() == 1 { - if NextYear(time_iterator, end_time) { - if time_iterator.Month() == 1 { - break - } else { - - month_id := GetTimeID(YM, uint(time_iterator.Year()), uint(time_iterator.Month()), 0, 0, tile_id) - results = append(results, month_id) - time_iterator = IncrementMonth(time_iterator) - } - } else { - break - } - } else { - day_id := GetTimeID(YMD, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), 0, tile_id) - results = append(results, day_id) - time_iterator = IncrementDay(time_iterator) - } - } else { - break - } - } else { - hour_id := GetTimeID(YMDH, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), uint(time_iterator.Hour()), tile_id) - results = append(results, hour_id) - time_iterator = IncrementHour(time_iterator) - } - } else { - break - } - } - return results, time_iterator - -} - -func downHill(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 { - var results []uint64 - time_iterator := start_time - for time_iterator.Before(end_time) { - if NextYear(time_iterator, end_time) { - year_id := GetTimeID(Y, uint(time_iterator.Year()), 0, 0, 0, tile_id) - results = append(results, year_id) - time_iterator = IncrementYear(time_iterator) - } else if NextMonth(time_iterator, end_time) { - month_id := GetTimeID(YM, uint(time_iterator.Year()), uint(time_iterator.Month()), 0, 0, tile_id) - results = append(results, month_id) - time_iterator = IncrementMonth(time_iterator) - } else if NextDay(time_iterator, end_time) { - day_id := GetTimeID(YMD, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), 0, tile_id) - results = append(results, day_id) - time_iterator = IncrementDay(time_iterator) - } else { - hour_id := GetTimeID(YMDH, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), uint(time_iterator.Hour()), tile_id) - results = append(results, hour_id) - time_iterator = IncrementHour(time_iterator) - } - } - return results -} diff --git a/timeframe_test.go b/timeframe_test.go deleted file mode 100644 index f31fcc32e..000000000 --- a/timeframe_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package pilosa_test - -import ( - "testing" - "time" - - "github.com/umbel/pilosa" -) - -func TestGetRange_1h_0(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-08-11 14:00"), - MustParseTime("2014-08-11 16:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h_1(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 10:03"), - MustParseTime("2014-01-02 11:03"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_2h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 10:03"), - MustParseTime("2014-01-02 12:03"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_24h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 12:03"), - MustParseTime("2014-01-03 12:03"), - uint64(1), - ); len(m) != 24 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1d(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 00:00"), - MustParseTime("2014-01-03 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1d1h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 00:00"), - MustParseTime("2014-01-03 01:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h1d(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 23:00"), - MustParseTime("2014-01-04 00:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h1d1h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 23:00"), - MustParseTime("2014-01-04 01:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1y(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-01 00:00"), - MustParseTime("2015-01-01 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h1d1m(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-30 23:00"), - MustParseTime("2014-03-01 00:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h1d1m1d1h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-30 23:00"), - MustParseTime("2014-03-02 01:00"), - uint64(1), - ); len(m) != 5 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetTimeIds(t *testing.T) { - _ = pilosa.GetTimeIds(uint64(15027), MustParseTime("1970-01-01 00:00"), pilosa.YMD) -} - -// DefaultTimeLayout is the time layout used by the tests. -const DefaultTimeLayout = "2006-01-02 15:04" - -// MustParseTime parses value using DefaultTimeLayout. Panic on error. -func MustParseTime(value string) time.Time { - v, err := time.Parse(DefaultTimeLayout, value) - if err != nil { - panic(err) - } - return v -}