diff --git a/attr.go b/attr.go index 0a2a0b5d2..38abd002e 100644 --- a/attr.go +++ b/attr.go @@ -112,6 +112,12 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { } switch v := v.(type) { + case int: + attr[k] = uint64(v) + case uint: + attr[k] = uint64(v) + case int64: + attr[k] = uint64(v) case string, uint64, bool: attr[k] = v default: diff --git a/attr_test.go b/attr_test.go index af421249c..c5a909642 100644 --- a/attr_test.go +++ b/attr_test.go @@ -15,9 +15,9 @@ func TestAttrStore_Attrs(t *testing.T) { defer s.Close() // Set attributes. - if err := s.SetAttrs(1, map[string]interface{}{"A": int64(100)}); err != nil { + if err := s.SetAttrs(1, map[string]interface{}{"A": uint64(100)}); err != nil { t.Fatal(err) - } else if err := s.SetAttrs(2, map[string]interface{}{"A": int64(200)}); err != nil { + } else if err := s.SetAttrs(2, map[string]interface{}{"A": uint64(200)}); err != nil { t.Fatal(err) } else if err := s.SetAttrs(1, map[string]interface{}{"B": "VALUE"}); err != nil { t.Fatal(err) @@ -26,14 +26,14 @@ func TestAttrStore_Attrs(t *testing.T) { // Retrieve attributes for profile #1. if m, err := s.Attrs(1); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE"}) { + } else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(100), "B": "VALUE"}) { t.Fatalf("unexpected attrs(1): %#v", m) } // Retrieve attributes for profile #2. if m, err := s.Attrs(2); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) { + } else if !reflect.DeepEqual(m, map[string]interface{}{"A": uint64(200)}) { t.Fatalf("unexpected attrs(2): %#v", m) } } diff --git a/cmd/pilosa/main_test.go b/cmd/pilosa/main_test.go index 9b99e3b72..6bdbcbef0 100644 --- a/cmd/pilosa/main_test.go +++ b/cmd/pilosa/main_test.go @@ -31,10 +31,8 @@ func TestMain_Set_Quick(t *testing.T) { // Execute SetBit() commands. for _, cmd := range cmds { - if res, err := m.Query("db=d", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil { + if _, err := m.Query("db=d", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil { t.Fatal(err) - } else if res != `{}`+"\n" { - t.Fatalf("unexpected result: %s", res) } } diff --git a/executor.go b/executor.go index caa816af1..d208c813a 100644 --- a/executor.go +++ b/executor.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "sort" + "time" "github.com/gogo/protobuf/proto" "github.com/umbel/pilosa/internal" @@ -41,16 +42,21 @@ func NewExecutor(index *Index) *Executor { func (e *Executor) Index() *Index { return e.index } // Execute executes a PQL query. -func (e *Executor) Execute(db string, q *pql.Query, slices []uint64) (interface{}, error) { +func (e *Executor) Execute(db string, q *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error) { // Verify that a database is set. if db == "" { return nil, ErrDatabaseRequired } + // Default options. + if opt == nil { + opt = &ExecOptions{} + } + // Ignore slices for set calls. switch root := q.Root.(type) { case *pql.SetBit: - return e.executeSetBit(db, root) + return e.executeSetBit(db, root, opt) case *pql.SetBitmapAttrs: return nil, e.executeSetBitmapAttrs(db, root) case *pql.SetProfileAttrs: @@ -70,27 +76,27 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64) (interface{ } } - return e.executeCall(db, q.Root, slices) + return e.executeCall(db, q.Root, slices, opt) } // executeCall executes a call. -func (e *Executor) executeCall(db string, c pql.Call, slices []uint64) (interface{}, error) { +func (e *Executor) executeCall(db string, c pql.Call, slices []uint64, opt *ExecOptions) (interface{}, error) { switch c := c.(type) { case pql.BitmapCall: - return e.executeBitmapCall(db, c, slices) + return e.executeBitmapCall(db, c, slices, opt) case *pql.Count: - return e.executeCount(db, c, slices) + return e.executeCount(db, c, slices, opt) case *pql.Profile: - return e.executeProfile(db, c) + return e.executeProfile(db, c, opt) case *pql.TopN: - return e.executeTopN(db, c, slices) + return e.executeTopN(db, c, slices, opt) default: panic("unreachable") } } // executeBitmapCall executes a call that returns a bitmap. -func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint64) (*Bitmap, error) { +func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint64, opt *ExecOptions) (*Bitmap, error) { other := NewBitmap() for node, nodeSlices := range e.slicesByNode(slices) { // Execute locally if the hostname matches. @@ -106,7 +112,7 @@ func (e *Executor) executeBitmapCall(db string, c pql.BitmapCall, slices []uint6 } // Otherwise execute remotely. - res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices) + res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt) if err != nil { return nil, err } @@ -147,7 +153,7 @@ func (e *Executor) executeBitmapCallSlice(db string, c pql.BitmapCall, slice uin } // executeTopN executes a TopN() call. -func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, error) { +func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64, opt *ExecOptions) ([]Pair, error) { var results []Pair for node, nodeSlices := range e.slicesByNode(slices) { // Execute locally if the hostname matches. @@ -163,7 +169,7 @@ func (e *Executor) executeTopN(db string, c *pql.TopN, slices []uint64) ([]Pair, } // Otherwise execute remotely. - res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices) + res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt) if err != nil { return nil, err } @@ -260,7 +266,16 @@ func (e *Executor) executeIntersectSlice(db string, c *pql.Intersect, slice uint // executeRangeSlice executes a range() call for a local slice. func (e *Executor) executeRangeSlice(db string, c *pql.Range, slice uint64) (*Bitmap, error) { - panic("FIXME") + frame := c.Frame + if frame == "" { + frame = DefaultFrame + } + + f := e.Index().Fragment(db, frame, slice) + if f == nil { + return NewBitmap(), nil + } + return f.Range(c.ID, c.StartTime, c.EndTime), nil } // executeUnionSlice executes a union() call for a local slice. @@ -283,7 +298,7 @@ func (e *Executor) executeUnionSlice(db string, c *pql.Union, slice uint64) (*Bi } // executeCount executes a count() call. -func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint64, error) { +func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64, opt *ExecOptions) (uint64, error) { var n uint64 for node, nodeSlices := range e.slicesByNode(slices) { // Execute locally if the hostname matches. @@ -299,7 +314,7 @@ func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint6 } // Otherwise execute remotely. - res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices) + res, err := e.exec(node, db, &pql.Query{Root: c}, nodeSlices, opt) if err != nil { return 0, err } @@ -310,12 +325,12 @@ func (e *Executor) executeCount(db string, c *pql.Count, slices []uint64) (uint6 // executeProfile executes a Profile() call. // This call only executes locally since the profile attibutes are stored locally. -func (e *Executor) executeProfile(db string, c *pql.Profile) (*Profile, error) { +func (e *Executor) executeProfile(db string, c *pql.Profile, opt *ExecOptions) (*Profile, error) { panic("FIXME: impl: e.Index().ProfileAttr(c.ID)") } // executeSetBit executes a SetBit() call. -func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) { +func (e *Executor) executeSetBit(db string, c *pql.SetBit, opt *ExecOptions) (bool, error) { slice := c.ProfileID / SliceWidth ret := false for _, node := range e.Cluster.SliceNodes(slice) { @@ -325,7 +340,7 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) { if err != nil { return false, fmt.Errorf("fragment: %s", err) } - val, err := f.SetBit(c.ID, c.ProfileID) + val, err := f.SetBit(c.ID, c.ProfileID, opt.Timestamp, opt.Quantum) if err != nil { return false, err } @@ -336,7 +351,7 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) { } // Forward call to remote node otherwise. - if res, err := e.exec(node, db, &pql.Query{Root: c}, nil); err != nil { + if res, err := e.exec(node, db, &pql.Query{Root: c}, nil, opt); err != nil { return false, err } else { ret = res.(bool) @@ -382,13 +397,18 @@ func (e *Executor) executeSetProfileAttrs(db string, c *pql.SetProfileAttrs) err } // exec executes a PQL query remotely for a set of slices on a node. -func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64) (result interface{}, err error) { +func (e *Executor) exec(node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (result interface{}, err error) { // Encode request object. - buf, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String(db), - Query: proto.String(q.String()), - Slices: slices, - }) + pbreq := &internal.QueryRequest{ + DB: proto.String(db), + Query: proto.String(q.String()), + Slices: slices, + Quantum: proto.Uint32(uint32(opt.Quantum)), + } + if opt.Timestamp != nil { + pbreq.Timestamp = proto.Int64(opt.Timestamp.UnixNano()) + } + buf, err := proto.Marshal(pbreq) if err != nil { return nil, err } @@ -465,6 +485,12 @@ func (e *Executor) slicesByNode(slices []uint64) map[*Node][]uint64 { return m } +// ExecOptions represents an execution context for a single Execute() call. +type ExecOptions struct { + Timestamp *time.Time + Quantum TimeQuantum +} + // decodeError returns an error representation of s if s is non-blank. // Returns nil if s is blank. func decodeError(s string) error { diff --git a/executor_test.go b/executor_test.go index 70e780c80..b55c408e7 100644 --- a/executor_test.go +++ b/executor_test.go @@ -17,12 +17,12 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 0).MustSetBits(10, 3) idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+1) - if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": 123}); err != nil { + if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -30,7 +30,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatalf("unexpected chunk(0): %s", spew.Sdump(chunks[0])) } else if chunks[1].Value[0] != 2 { t.Fatalf("unexpected chunk(1): %s", spew.Sdump(chunks[1])) - } else if attrs := res.(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": 123}) { + } else if attrs := res.(*pilosa.Bitmap).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": uint64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } } @@ -45,7 +45,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "general", 0).MustSetBits(11, 2) e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Difference(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 1 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -67,7 +67,7 @@ func TestExecutor_Execute_Intersect(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Intersect(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -90,7 +90,7 @@ func TestExecutor_Execute_Union(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "general", 1).MustSetBits(11, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Union(Bitmap(id=10), Bitmap(id=11))`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 2 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -110,7 +110,7 @@ func TestExecutor_Execute_Count(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, SliceWidth+2) e := NewExecutor(idx.Index, NewCluster(1)) - if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil { + if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if n != uint64(3) { t.Fatalf("unexpected n: %d", n) @@ -156,23 +156,23 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := NewExecutor(idx.Index, NewCluster(1)) - if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=XXX, YYY=1)`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=XXX, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } f := idx.Frame("d", "f") if m, err := f.BitmapAttrStore().Attrs(10); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { + } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": uint64(123), "bat": true}) { t.Fatalf("unexpected bitmap attr: %#v", m) } } @@ -183,19 +183,19 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2) - idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0, nil, 0) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) - if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil); err != nil { + if result, err := e.Execute("d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []pilosa.Pair{ {Key: 0, Count: 5}, @@ -211,23 +211,23 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2, nil, 0) // Create an intersecting bitmap. - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2, nil, 0) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) - if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil); err != nil { + if result, err := e.Execute("d", MustParse(`TopN(Bitmap(id=100, frame=other), frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(result, []pilosa.Pair{ {Key: 20, Count: 3}, @@ -238,6 +238,24 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { } } +// Ensure a range query can be executed. +func TestExecutor_Execute_Range(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + f := idx.MustCreateFragmentIfNotExists("d", "f.t", 0) + if _, err := f.SetBit(1, 100, MustParseTime("2000-01-01 00:00"), pilosa.YMD); err != nil { + t.Fatal(err) + } + + e := NewExecutor(idx.Index, NewCluster(1)) + if res, err := e.Execute("d", MustParse(`Range(id=1, frame=f.t, start="2000-01-01T00:00", end="2000-01-01T01:00")`), nil, nil); err != nil { + t.Fatal(err) + } else if bits := res.(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) { + t.Fatalf("unexpected bits: %+v", bits) + } +} + // Ensure a remote query can return a bitmap. func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { c := NewCluster(2) @@ -248,7 +266,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Bitmap(id=10, frame=f)` { @@ -273,7 +291,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+1) e := NewExecutor(idx.Index, c) - if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`Bitmap(id=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) } else if chunks := res.(*pilosa.Bitmap).Chunks(); len(chunks) != 3 { t.Fatalf("unexpected chunk length: %s", spew.Sdump(chunks)) @@ -294,7 +312,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to return a count. - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return uint64(10), nil } @@ -305,7 +323,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 1).MustSetBits(10, (1*SliceWidth)+2) e := NewExecutor(idx.Index, c) - if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil); err != nil { + if n, err := e.Execute("d", MustParse(`Count(Bitmap(id=10, frame=f))`), nil, nil); err != nil { t.Fatal(err) } else if n != uint64(12) { t.Fatalf("unexpected n: %d", n) @@ -324,7 +342,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { // Mock secondary server's executor to verify arguments. var remoteCalled bool - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) } else if query.String() != `SetBit(id=10, frame=f, profileID=2)` { @@ -339,7 +357,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { defer idx.Close() e := NewExecutor(idx.Index, c) - if _, err := e.Execute("d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil); err != nil { + if _, err := e.Execute("d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil { t.Fatal(err) } @@ -362,7 +380,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { c.Nodes[1].Host = s.Host() // Mock secondary server's executor to verify arguments and return a bitmap. - s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + s.Handler.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) } else if query.String() != `TopN(frame=f, n=3)` { @@ -386,7 +404,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", 3).MustSetBits(30, (3*SliceWidth)+2) e := NewExecutor(idx.Index, c) - if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil); err != nil { + if res, err := e.Execute("d", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(res, []pilosa.Pair{ {Key: 0, Count: 5}, diff --git a/fragment.go b/fragment.go index 8d60c0004..6945261f5 100644 --- a/fragment.go +++ b/fragment.go @@ -188,7 +188,7 @@ func (f *Fragment) openStorage() error { // openCache initializes the cache from bitmap ids persisted to disk. func (f *Fragment) openCache() error { // Determine cache type from frame name. - if strings.HasSuffix(f.frame, ".n") { + if strings.HasSuffix(f.frame, FrameSuffixRank) { c := NewRankCache() c.ThresholdLength = 50000 c.ThresholdIndex = 45000 @@ -312,13 +312,19 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { // SetBit sets a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) { +func (f *Fragment) SetBit(bitmapID, profileID uint64, t *time.Time, q TimeQuantum) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() + + // Set time bits if this is a time-frame and a timestamp is specified. + if strings.HasSuffix(f.frame, FrameSuffixTime) && t != nil { + return f.setTimeBit(bitmapID, profileID, *t, q) + } + return f.setBit(bitmapID, profileID) } -func (f *Fragment) setBit(bitmapID, profileID uint64) (bool, error) { +func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) { // Determine the position of the bit in the storage. ret := false pos, err := f.pos(bitmapID, profileID) @@ -340,6 +346,17 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (bool, error) { } +func (f *Fragment) setTimeBit(bitmapID, profileID uint64, t time.Time, q TimeQuantum) (changed bool, err error) { + for _, timeID := range TimeIDsFromQuantum(q, t, bitmapID) { + if v, err := f.setBit(timeID, profileID); err != nil { + return changed, fmt.Errorf("set time bit: t=%s, q=%s, err=%s", t, q, err) + } else if v { + changed = true + } + } + return changed, nil +} + // ClearBit clears a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) { @@ -481,16 +498,18 @@ func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap { f.mu.Lock() defer f.mu.Unlock() - bitmapIDs := GetRange(start, end, bitmapID) + // Retrieve a list of bitmap ids for a given time range. + bitmapIDs := TimeIDsFromRange(start, end, bitmapID) if len(bitmapIDs) == 0 { return NewBitmap() } - result := f.bitmap(bitmapIDs[0]) + // Union all bitmap ids from the time range. + bm := f.bitmap(bitmapIDs[0]) for _, id := range bitmapIDs[1:] { - result = result.Union(f.bitmap(id)) + bm = bm.Union(f.bitmap(id)) } - return result + return bm } // Import bulk imports a set of bits and then snapshots the storage. diff --git a/fragment_test.go b/fragment_test.go index 441c254e5..e5903c124 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -19,11 +19,11 @@ func TestFragment_SetBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(120, 1); err != nil { + if _, err := f.SetBit(120, 1, nil, 0); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 6); err != nil { + } else if _, err := f.SetBit(120, 6, nil, 0); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(121, 0); err != nil { + } else if _, err := f.SetBit(121, 0, nil, 0); err != nil { t.Fatal(err) } @@ -50,9 +50,9 @@ func TestFragment_ClearBit(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.SetBit(1000, 1, nil, 0); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -77,9 +77,9 @@ func TestFragment_Snapshot(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1); err != nil { + if _, err := f.SetBit(1000, 1, nil, 0); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -133,11 +133,11 @@ func TestFragment_TopN_Filter(t *testing.T) { f.MustSetBits(102, 1, 2) // Assign attributes. - f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": 10}) - f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": 20}) + f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) + f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)}) // Retrieve top bitmaps. - if pairs, err := f.TopN(2, nil, "x", []interface{}{10, 15, 20}); err != nil { + if pairs, err := f.TopN(2, nil, "x", []interface{}{uint64(10), uint64(15), uint64(20)}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { t.Fatalf("unexpected count: %d", len(pairs)) @@ -176,6 +176,10 @@ func TestFragment_TopN_Intersect(t *testing.T) { // Ensure a fragment can return top bitmaps that have many bits set. func TestFragment_TopN_Intersect_Large(t *testing.T) { + if testing.Short() { + t.Skip("short mode") + } + f := MustOpenFragment("d", "f", 0) defer f.Close() @@ -218,7 +222,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.SetBit(i, 0, nil, 0); err != nil { t.Fatal(err) } } @@ -250,7 +254,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0); err != nil { + if _, err := f.SetBit(i, 0, nil, 0); err != nil { t.Fatal(err) } } @@ -330,9 +334,10 @@ func (f *Fragment) Reopen() error { } // MustSetBits sets bits on a bitmap. Panic on error. +// This function does not accept a timestamp or quantum. func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { for _, profileID := range profileIDs { - if _, err := f.SetBit(bitmapID, profileID); err != nil { + if _, err := f.SetBit(bitmapID, profileID, nil, 0); err != nil { panic(err) } } diff --git a/frame.go b/frame.go index 6142f5ec7..9dd48d615 100644 --- a/frame.go +++ b/frame.go @@ -8,6 +8,14 @@ import ( "sync" ) +const ( + // FrameSuffixTime is the suffix used for time-based frames. + FrameSuffixTime = ".t" + + // FrameSuffixRank is the suffix used for rank-based frames. + FrameSuffixRank = ".n" +) + // Frame represents a container for fragments. type Frame struct { mu sync.Mutex diff --git a/handler.go b/handler.go index a1e65878f..24201b84c 100644 --- a/handler.go +++ b/handler.go @@ -29,7 +29,7 @@ type Handler struct { // The execution engine for running queries. Executor interface { - Execute(db string, query *pql.Query, slices []uint64) (interface{}, error) + Execute(db string, query *pql.Query, slices []uint64, opt *ExecOptions) (interface{}, error) } // The version to report on the /version endpoint. @@ -95,6 +95,12 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { return } + // Build execution options. + opt := &ExecOptions{ + Timestamp: req.Timestamp, + Quantum: req.Quantum, + } + // Parse query string. q, err := pql.NewParser(strings.NewReader(req.Query)).Parse() if err != nil { @@ -104,7 +110,7 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } // Execute the query. - result, err := h.Executor.Execute(req.DB, q, req.Slices) + result, err := h.Executor.Execute(req.DB, q, req.Slices, opt) resp := &QueryResponse{Result: result, Err: err} // Fill profile attributes if requested. @@ -196,11 +202,38 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { return nil, errors.New("invalid slice argument") } + // Parse timestamp, if available. + var timestamp *time.Time + if v := q.Get("timestamp"); v != "" { + layout := "2006-01-02 15:04:05" + if strings.Contains(v, "T") { + layout = "2006-01-02T15:04:05" + } + + t, err := time.Parse(layout, v) + if err != nil { + return nil, errors.New("invalid timestamp") + } + timestamp = &t + } + + // Parse time granularity. + quantum := YMDH + if s := q.Get("time_granularity"); s != "" { + v, err := ParseTimeQuantum(s) + if err != nil { + return nil, errors.New("invalid time granularity") + } + quantum = v + } + return &QueryRequest{ - DB: q.Get("db"), - Query: query, - Slices: slices, - Profiles: q.Get("profiles") == "true", + DB: q.Get("db"), + Query: query, + Slices: slices, + Profiles: q.Get("profiles") == "true", + Timestamp: timestamp, + Quantum: quantum, }, nil } @@ -352,15 +385,29 @@ type QueryRequest struct { // Return profile attributes, if true. Profiles bool + + // Timestamp passed into the query. + Timestamp *time.Time + + // Time granularity to use with the timestamp. + Quantum TimeQuantum } func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { - return &QueryRequest{ + req := &QueryRequest{ DB: pb.GetDB(), Query: pb.GetQuery(), Slices: pb.GetSlices(), Profiles: pb.GetProfiles(), + Quantum: TimeQuantum(pb.GetQuantum()), } + + if pb.Timestamp != nil { + t := time.Unix(0, pb.GetTimestamp()) + req.Timestamp = &t + } + + return req } // QueryResponse represent a response from a processed query. diff --git a/handler_test.go b/handler_test.go index a8c3a64f8..65d000c3e 100644 --- a/handler_test.go +++ b/handler_test.go @@ -29,7 +29,7 @@ func TestHandler_NotFound(t *testing.T) { // Ensure the handler can accept URL arguments. func TestHandler_Query_Args_URL(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != "db0" { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Count(Bitmap(id=100))` { @@ -52,7 +52,7 @@ func TestHandler_Query_Args_URL(t *testing.T) { // Ensure the handler can accept arguments via protobufs. func TestHandler_Query_Args_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { if db != "db0" { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Count(Bitmap(id=100))` { @@ -98,7 +98,7 @@ func TestHandler_Query_Args_Err(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as JSON. func TestHandler_Query_Uint64_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return uint64(100), nil } @@ -114,7 +114,7 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) { // Ensure the handler can execute a query with a uint64 response as protobufs. func TestHandler_Query_Uint64_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return uint64(100), nil } @@ -137,7 +137,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as JSON. func TestHandler_Query_Bitmap_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} return bm, nil @@ -169,7 +169,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { h := NewHandler() h.Index = idx.Index - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true} return bm, nil @@ -187,7 +187,7 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { // Ensure the handler can execute a query that returns a bitmap as protobuf. func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} return bm, nil @@ -232,7 +232,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { h := NewHandler() h.Index = idx.Index - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1) bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true} return bm, nil @@ -287,7 +287,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Ensure the handler can execute a query that returns pairs as JSON. func TestHandler_Query_Pairs_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return []pilosa.Pair{ {Key: 1, Count: 2}, {Key: 3, Count: 4}, @@ -306,7 +306,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) { // Ensure the handler can execute a query that returns pairs as protobuf. func TestHandler_Query_Pairs_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return []pilosa.Pair{ {Key: 1, Count: 2}, {Key: 3, Count: 4}, @@ -332,7 +332,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) { // Ensure the handler can return an error as JSON. func TestHandler_Query_Err_JSON(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return nil, errors.New("marker") } @@ -348,7 +348,7 @@ func TestHandler_Query_Err_JSON(t *testing.T) { // Ensure the handler can return an error as protobuf. func TestHandler_Query_Err_Protobuf(t *testing.T) { h := NewHandler() - h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64) (interface{}, error) { + h.Executor.ExecuteFn = func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { return nil, errors.New("marker") } @@ -449,13 +449,13 @@ func NewHandler() *Handler { // HandlerExecutor is a mock implementing pilosa.Handler.Executor. type HandlerExecutor struct { cluster *pilosa.Cluster - ExecuteFn func(db string, query *pql.Query, slices []uint64) (interface{}, error) + ExecuteFn func(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) } func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster } -func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64) (interface{}, error) { - return c.ExecuteFn(db, query, slices) +func (c *HandlerExecutor) Execute(db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) (interface{}, error) { + return c.ExecuteFn(db, query, slices, opt) } // Server represents a test wrapper for httptest.Server. diff --git a/internal/internal.pb.go b/internal/internal.pb.go index 82d20783f..cf20adead 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -25,7 +25,6 @@ It has these top-level messages: package internal import proto "github.com/gogo/protobuf/proto" - import fmt "fmt" import math "math" @@ -222,6 +221,8 @@ type QueryRequest struct { Query *string `protobuf:"bytes,2,req,name=Query" json:"Query,omitempty"` Slices []uint64 `protobuf:"varint,3,rep,name=Slices" json:"Slices,omitempty"` Profiles *bool `protobuf:"varint,4,opt,name=Profiles" json:"Profiles,omitempty"` + Timestamp *int64 `protobuf:"varint,5,opt,name=Timestamp" json:"Timestamp,omitempty"` + Quantum *uint32 `protobuf:"varint,6,opt,name=Quantum" json:"Quantum,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -258,6 +259,20 @@ func (m *QueryRequest) GetProfiles() bool { return false } +func (m *QueryRequest) GetTimestamp() int64 { + if m != nil && m.Timestamp != nil { + return *m.Timestamp + } + return 0 +} + +func (m *QueryRequest) GetQuantum() uint32 { + if m != nil && m.Quantum != nil { + return *m.Quantum + } + return 0 +} + type QueryResponse struct { Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"` Bitmap *Bitmap `protobuf:"bytes,2,opt,name=Bitmap" json:"Bitmap,omitempty"` @@ -414,31 +429,33 @@ func init() { } var fileDescriptor0 = []byte{ - // 410 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x84, 0x92, 0xc1, 0x8e, 0xda, 0x30, - 0x10, 0x86, 0x15, 0xe2, 0x04, 0x32, 0x29, 0x01, 0xdc, 0x0b, 0xaa, 0x84, 0x8a, 0xcc, 0x05, 0xf5, - 0xc0, 0x01, 0xf5, 0x05, 0x0a, 0xb4, 0x2a, 0x42, 0x45, 0xb4, 0xa8, 0x3d, 0x37, 0xa2, 0x2e, 0x44, - 0x0d, 0x71, 0xea, 0x38, 0x07, 0x5e, 0xa6, 0xcf, 0xda, 0xb1, 0xe3, 0x04, 0x76, 0x97, 0xd5, 0x9e, - 0x22, 0xff, 0x1e, 0xcf, 0x7c, 0xff, 0xfc, 0x81, 0x28, 0xc9, 0x14, 0x97, 0x59, 0x9c, 0xce, 0x72, - 0x29, 0x94, 0xa0, 0x9d, 0xfa, 0xcc, 0x3e, 0x83, 0xbf, 0x48, 0xd4, 0x39, 0xce, 0xe9, 0x5b, 0xf0, - 0x97, 0xa7, 0x32, 0xfb, 0x53, 0x0c, 0x9d, 0xb1, 0x3b, 0x0d, 0xe7, 0xbd, 0x59, 0xf3, 0xc8, 0xe8, - 0x74, 0x04, 0xde, 0x07, 0xa5, 0x64, 0x31, 0x6c, 0x99, 0xfb, 0xe8, 0x7a, 0xaf, 0x65, 0x36, 0x01, - 0xaf, 0xaa, 0x0b, 0xc1, 0xdd, 0xf0, 0x0b, 0x76, 0x69, 0x4d, 0x09, 0xed, 0x82, 0xf7, 0x23, 0x4e, - 0x4b, 0x6e, 0x1e, 0x11, 0xc6, 0x80, 0xec, 0xe2, 0x44, 0x3e, 0xa9, 0x59, 0x8a, 0x32, 0x53, 0x58, - 0x83, 0x47, 0xf6, 0x0e, 0x5c, 0x44, 0xa2, 0x7d, 0xe8, 0x54, 0x64, 0xeb, 0x95, 0xad, 0x1b, 0x40, - 0xb0, 0x93, 0xe2, 0x77, 0x92, 0x72, 0x94, 0xaa, 0xda, 0xf7, 0xd0, 0xb6, 0x12, 0x05, 0x68, 0x35, - 0x95, 0x2f, 0xa0, 0x6e, 0x81, 0xe8, 0xef, 0x2d, 0x45, 0x40, 0x5f, 0x43, 0xb8, 0x57, 0x32, 0xc9, - 0x8e, 0x35, 0xaf, 0x83, 0x22, 0x8e, 0xfc, 0x8e, 0x6f, 0x2b, 0xc9, 0x45, 0xc9, 0x50, 0x2c, 0x84, - 0x48, 0x2b, 0x89, 0xa0, 0xd4, 0x61, 0x53, 0x68, 0xeb, 0x7e, 0x5f, 0x70, 0x8b, 0xcd, 0x64, 0xe7, - 0xee, 0xe4, 0x0d, 0xbc, 0xfa, 0x5a, 0x72, 0x79, 0xf9, 0xc6, 0xff, 0x96, 0xbc, 0x50, 0x1a, 0x7a, - 0xb5, 0xb0, 0x00, 0xb8, 0x06, 0x73, 0x67, 0xac, 0x05, 0x34, 0x02, 0x7f, 0x9f, 0x26, 0x07, 0x5e, - 0xe0, 0x5c, 0x5c, 0x9d, 0xde, 0x87, 0xb5, 0x5a, 0xd8, 0xb1, 0xff, 0x1c, 0xe8, 0xda, 0x6e, 0x45, - 0x2e, 0xb2, 0x82, 0x6b, 0x43, 0x1f, 0xa5, 0xc4, 0x7e, 0x9a, 0x7d, 0x5c, 0x47, 0x6b, 0xbc, 0x84, - 0xf3, 0xfe, 0x95, 0xc5, 0x46, 0x1e, 0x80, 0xb3, 0xb5, 0xae, 0x90, 0x5b, 0x07, 0xa3, 0x5b, 0x3f, - 0xe2, 0x36, 0x79, 0x4d, 0x6e, 0x86, 0x7b, 0xa6, 0x62, 0x70, 0x53, 0x61, 0x13, 0xe8, 0x41, 0x7b, - 0x79, 0x8a, 0xb3, 0x23, 0xff, 0x35, 0xf4, 0x0d, 0xe0, 0x4f, 0xe8, 0xae, 0xcf, 0xb9, 0x90, 0xea, - 0x19, 0xbb, 0x9f, 0x64, 0x7c, 0xe6, 0xd6, 0x2e, 0x1e, 0x8d, 0x5d, 0xe4, 0xb1, 0x59, 0xd7, 0xe9, - 0x57, 0x4c, 0x84, 0xe2, 0xeb, 0x26, 0xfe, 0x8a, 0x82, 0xb0, 0x11, 0x44, 0xf5, 0x84, 0x3b, 0x2b, - 0x60, 0x6f, 0xf0, 0xcf, 0x8a, 0x0f, 0x27, 0xfe, 0xb0, 0x9d, 0x8e, 0x86, 0xfc, 0x0f, 0x00, 0x00, - 0xff, 0xff, 0x98, 0x36, 0x6d, 0xac, 0x14, 0x03, 0x00, 0x00, + // 436 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x84, 0x92, 0xcf, 0x6e, 0xd4, 0x30, + 0x10, 0xc6, 0x95, 0x8d, 0x93, 0x4d, 0x26, 0x24, 0x6d, 0xcd, 0x25, 0x42, 0xaa, 0xa8, 0xdc, 0xcb, + 0x8a, 0x43, 0x0f, 0x15, 0x2f, 0xc0, 0x6e, 0x41, 0x54, 0x88, 0xaa, 0xa5, 0xc0, 0x19, 0xab, 0x98, + 0x6e, 0x44, 0x62, 0x07, 0xc7, 0x39, 0xf4, 0x65, 0x78, 0x56, 0xc6, 0x7f, 0x92, 0x2e, 0xb0, 0x88, + 0x53, 0x94, 0xcf, 0x63, 0x7f, 0xbf, 0xf9, 0x66, 0xa0, 0x6a, 0xa4, 0x11, 0x5a, 0xf2, 0xf6, 0xac, + 0xd7, 0xca, 0x28, 0x9a, 0x4d, 0xff, 0xec, 0x2d, 0xa4, 0xeb, 0xc6, 0x74, 0xbc, 0xa7, 0xcf, 0x21, + 0xdd, 0x6c, 0x47, 0xf9, 0x7d, 0xa8, 0xa3, 0x93, 0x78, 0x55, 0x9c, 0x1f, 0x9c, 0xcd, 0x97, 0x9c, + 0x4e, 0x8f, 0x21, 0x79, 0x65, 0x8c, 0x1e, 0xea, 0x85, 0x3b, 0xaf, 0x1e, 0xcf, 0xad, 0xcc, 0x4e, + 0x21, 0xf1, 0x75, 0x05, 0xc4, 0xef, 0xc4, 0x03, 0xbe, 0xb2, 0x58, 0x11, 0x5a, 0x42, 0xf2, 0x99, + 0xb7, 0xa3, 0x70, 0x97, 0x08, 0x63, 0x40, 0xae, 0x79, 0xa3, 0xff, 0xaa, 0xd9, 0xa8, 0x51, 0x1a, + 0xac, 0xc1, 0x5f, 0xf6, 0x02, 0x62, 0x44, 0xa2, 0x87, 0x90, 0x79, 0xb2, 0xcb, 0x8b, 0x50, 0x77, + 0x04, 0xf9, 0xb5, 0x56, 0xdf, 0x9a, 0x56, 0xa0, 0xe4, 0x6b, 0x5f, 0xc2, 0x32, 0x48, 0x14, 0x60, + 0x31, 0x57, 0xfe, 0x07, 0xf5, 0x0a, 0x88, 0xfd, 0xee, 0x52, 0xe4, 0xf4, 0x29, 0x14, 0xb7, 0x46, + 0x37, 0xf2, 0x7e, 0xe2, 0x8d, 0x50, 0x44, 0xcb, 0x4f, 0x78, 0xd7, 0x4b, 0x31, 0x4a, 0x8e, 0x62, + 0xad, 0x54, 0xeb, 0x25, 0x82, 0x52, 0xc6, 0x56, 0xb0, 0xb4, 0xef, 0xbd, 0xc7, 0x14, 0x67, 0xe7, + 0x68, 0xaf, 0xb3, 0x82, 0x27, 0x37, 0xa3, 0xd0, 0x0f, 0x1f, 0xc4, 0x8f, 0x51, 0x0c, 0xc6, 0x42, + 0x5f, 0xac, 0x03, 0x00, 0xc6, 0xe0, 0xce, 0x5c, 0x6b, 0x39, 0xad, 0x20, 0xbd, 0x6d, 0x9b, 0x3b, + 0x31, 0xa0, 0x2f, 0x46, 0x67, 0xf3, 0x08, 0xad, 0x0e, 0xde, 0xd6, 0x92, 0x7c, 0x6c, 0x3a, 0x7c, + 0x86, 0x77, 0x7d, 0x9d, 0xa0, 0x14, 0xd3, 0x03, 0x58, 0xde, 0x8c, 0x5c, 0x9a, 0xb1, 0xab, 0x53, + 0x14, 0x4a, 0xf6, 0x33, 0x82, 0x32, 0x38, 0x0e, 0xbd, 0x92, 0x83, 0xb0, 0x4d, 0xbf, 0xd6, 0x1a, + 0x3d, 0x6d, 0x7f, 0x27, 0xd3, 0xf8, 0x5d, 0xbf, 0xc5, 0xf9, 0xe1, 0x23, 0x6f, 0x58, 0x8b, 0x1c, + 0xa2, 0xab, 0xd0, 0x39, 0xf6, 0x66, 0x87, 0x67, 0xed, 0xff, 0xe8, 0xcd, 0xcd, 0xf4, 0x74, 0x07, + 0x30, 0x71, 0x15, 0x47, 0x3b, 0x15, 0x61, 0x4a, 0x08, 0xb8, 0xd9, 0x72, 0x79, 0x2f, 0xbe, 0x3a, + 0xc0, 0x8c, 0x7d, 0x81, 0xf2, 0xb2, 0xeb, 0x95, 0x36, 0xff, 0x88, 0xe4, 0x8d, 0xe6, 0x9d, 0x08, + 0x91, 0xe0, 0xaf, 0x8b, 0x04, 0x79, 0xc2, 0x3e, 0x4c, 0x1b, 0xe2, 0x99, 0x08, 0xc5, 0xdb, 0xf3, + 0x8a, 0x78, 0x0a, 0xc2, 0x8e, 0xa1, 0x9a, 0x1c, 0xf6, 0x44, 0xc0, 0x9e, 0xe1, 0xf6, 0xf1, 0xbb, + 0xad, 0xf8, 0xfd, 0x39, 0x3b, 0x3e, 0xf2, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x77, 0xe5, 0x8e, 0x3b, + 0x38, 0x03, 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index 91ebb3128..26a87bbf2 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -37,10 +37,12 @@ message AttrMap { } message QueryRequest { - required string DB = 1; - required string Query = 2; - repeated uint64 Slices = 3; - optional bool Profiles = 4; + required string DB = 1; + required string Query = 2; + repeated uint64 Slices = 3; + optional bool Profiles = 4; + optional int64 Timestamp = 5; + optional uint32 Quantum = 6; } message QueryResponse { diff --git a/pql/parser.go b/pql/parser.go index 5f7025121..4761aeb88 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -441,7 +441,7 @@ func (p *Parser) parseSetProfileAttrsCall() (*SetProfileAttrs, error) { case string, bool: c.Attrs[key] = v case uint64: - c.Attrs[key] = int64(v) + c.Attrs[key] = v default: return nil, parseErrorf(pos, "invalid SetProfileAttrs() arg: %v", arg.key) } diff --git a/pql/parser_test.go b/pql/parser_test.go index ad4e3a1aa..df4bcf810 100644 --- a/pql/parser_test.go +++ b/pql/parser_test.go @@ -222,7 +222,7 @@ func TestParser_Parse_SetBitmapAttrs_Key(t *testing.T) { Frame: "b.n", Attrs: map[string]interface{}{ "foo": "bar", - "bar": int64(123), + "bar": uint64(123), "baz": true, "bat": false, "x": nil, @@ -244,7 +244,7 @@ func TestParser_Parse_SetBitmapAttrs_Array(t *testing.T) { Frame: "b.n", Attrs: map[string]interface{}{ "foo": "bar", - "bar": int64(123), + "bar": uint64(123), }, }, }) { diff --git a/time.go b/time.go new file mode 100644 index 000000000..ce11ea1d9 --- /dev/null +++ b/time.go @@ -0,0 +1,157 @@ +package pilosa + +import ( + "errors" + "strings" + "time" +) + +// TimeQuantum represents a time granularity for time-based bitmap ids. +type TimeQuantum uint + +// ParseTimeQuantum parses s into a quantum. +func ParseTimeQuantum(s string) (TimeQuantum, error) { + switch strings.ToUpper(s) { + case "Y": + return Y, nil + case "M": + return YM, nil + case "D": + return YMD, nil + case "H": + return YMDH, nil + default: + return 0, errors.New("invalid quantum") + } +} + +// String returns the string representation of the quantum. +func (q TimeQuantum) String() string { + switch q { + case Y: + return "Y" + case YM: + return "M" + case YMD: + return "D" + case YMDH: + return "H" + default: + return "Y" + } +} + +const ( + Y TimeQuantum = 3 + YM TimeQuantum = 2 + YMD TimeQuantum = 1 + YMDH TimeQuantum = 0 +) + +// TimeID returns a packed time identifier from a year/month/day/hour & tile. +func TimeID(q TimeQuantum, year uint, month uint, day uint, hour uint, tileID uint64) uint64 { + v := uint64((uint(q) << 30) | ((year - 1970) << 23) | (month << 19) | (day << 14) | (hour << 9)) + return (v << 32) | tileID +} + +// TimeIDsFromRange returns timestamp bitmap ids within a time range. +func TimeIDsFromRange(start, end time.Time, tileID uint64) []uint64 { + t := start + + var results []uint64 + for t.Before(end) { + if !nextDay(t, end) { + break + } + if t.Hour() == 0 { + if !nextMonth(t, end) { + break + } + + if t.Day() == 1 { + if !nextYear(t, end) { + break + } + + if t.Month() == 1 { + break + } + + results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) + t = t.AddDate(0, 1, 0) + } else { + results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) + t = t.AddDate(0, 0, 1) + } + } else { + results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) + t = t.Add(time.Hour) + } + } + + for t.Before(end) { + if nextYear(t, end) { + results = append(results, TimeID(Y, uint(t.Year()), 0, 0, 0, tileID)) + t = t.AddDate(1, 0, 0) + } else if nextMonth(t, end) { + results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) + t = t.AddDate(0, 1, 0) + } else if nextDay(t, end) { + results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) + t = t.AddDate(0, 0, 1) + } else { + results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) + t = t.Add(time.Hour) + } + } + + return results +} + +func nextYear(start time.Time, end time.Time) bool { + next := start.AddDate(1, 0, 0) + if next.Year() == end.Year() { + return true + } + return end.After(next) +} + +func nextMonth(start time.Time, end time.Time) bool { + next := start.AddDate(0, 1, 0) + y1, m1, _ := next.Date() + y2, m2, _ := end.Date() + if (y1 == y2) && (m1 == m2) { + return true + } + return end.After(next) +} + +func nextDay(start time.Time, end time.Time) bool { + next := start.AddDate(0, 0, 1) + y1, m1, d1 := next.Date() + y2, m2, d2 := end.Date() + if (y1 == y2) && (m1 == m2) && (d1 == d2) { + return true + } + return end.After(next) +} + +// TimeIDsFromQuantum returns a list of time identifiers for a single quantum. +func TimeIDsFromQuantum(q TimeQuantum, t time.Time, tileID uint64) []uint64 { + y, m, d, h := uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()) + + v := make([]uint64, 0, 4) + if q <= Y { + v = append(v, TimeID(Y, y, 0, 0, 0, tileID)) + } + if q <= YM { + v = append(v, TimeID(YM, y, m, 0, 0, tileID)) + } + if q <= YMD { + v = append(v, TimeID(YMD, y, m, d, 0, tileID)) + } + if q <= YMDH { + v = append(v, TimeID(YMDH, y, m, d, h, tileID)) + } + return v +} diff --git a/time_test.go b/time_test.go new file mode 100644 index 000000000..2d7644841 --- /dev/null +++ b/time_test.go @@ -0,0 +1,134 @@ +package pilosa_test + +import ( + "testing" + "time" + + "github.com/umbel/pilosa" +) + +func TestTimeIDsFromRange_1h_0(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-08-11 14:00"), + *MustParseTime("2014-08-11 16:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h_1(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 10:03"), + *MustParseTime("2014-01-02 11:03"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_2h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 10:03"), + *MustParseTime("2014-01-02 12:03"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_24h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 12:03"), + *MustParseTime("2014-01-03 12:03"), + uint64(1), + ); len(m) != 24 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1d(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 00:00"), + *MustParseTime("2014-01-03 00:00"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1d1h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 00:00"), + *MustParseTime("2014-01-03 01:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h1d(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 23:00"), + *MustParseTime("2014-01-04 00:00"), + uint64(1), + ); len(m) != 2 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h1d1h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-02 23:00"), + *MustParseTime("2014-01-04 01:00"), + uint64(1), + ); len(m) != 3 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1y(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-01 00:00"), + *MustParseTime("2015-01-01 00:00"), + uint64(1), + ); len(m) != 1 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h1d1m(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-30 23:00"), + *MustParseTime("2014-03-01 00:00"), + uint64(1), + ); len(m) != 3 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromRange_1h1d1m1d1h(t *testing.T) { + if m := pilosa.TimeIDsFromRange( + *MustParseTime("2014-01-30 23:00"), + *MustParseTime("2014-03-02 01:00"), + uint64(1), + ); len(m) != 5 { + t.Fatalf("unexpected range len: %d", len(m)) + } +} + +func TestTimeIDsFromQuantum(t *testing.T) { + _ = pilosa.TimeIDsFromQuantum(pilosa.YMD, *MustParseTime("1970-01-01 00:00"), uint64(15027)) +} + +// DefaultTimeLayout is the time layout used by the tests. +const DefaultTimeLayout = "2006-01-02 15:04" + +// MustParseTime parses value using DefaultTimeLayout. Panic on error. +func MustParseTime(value string) *time.Time { + v, err := time.Parse(DefaultTimeLayout, value) + if err != nil { + panic(err) + } + return &v +} diff --git a/timeframe.go b/timeframe.go deleted file mode 100644 index 22c75cc11..000000000 --- a/timeframe.go +++ /dev/null @@ -1,172 +0,0 @@ -package pilosa - -import ( - "time" -) - -type TimeQuantum uint - -const ( - Y TimeQuantum = 3 - YM TimeQuantum = 2 - YMD TimeQuantum = 1 - YMDH TimeQuantum = 0 -) - -func GetTimeID(t TimeQuantum, year uint, month uint, day uint, hour uint, tile_id uint64) uint64 { - y := year - 1970 //config.startyear - time_stamp := uint64((uint(t) << 30) | (y << 23) | (month << 19) | (day << 14) | (hour << 9)) - time_stamp = time_stamp << 32 - result := time_stamp | tile_id - - return result -} - -func GetTimeIds(tile_id uint64, atime time.Time, min_quantum TimeQuantum) []uint64 { - results := make([]uint64, 0, 4) - year := uint(atime.Year()) - month := atime.Month() - day := atime.Day() - hour := atime.Hour() - - if min_quantum <= Y { - id := GetTimeID(Y, year, 0, 0, 0, tile_id) - results = append(results, id) - } - - if min_quantum <= YM { - id := GetTimeID(YM, uint(year), uint(month), 0, 0, tile_id) - results = append(results, id) - } - - if min_quantum <= YMD { - id := GetTimeID(YMD, year, uint(month), uint(day), 0, tile_id) - results = append(results, id) - } - - if min_quantum <= YMDH { - id := GetTimeID(YMDH, year, uint(month), uint(day), uint(hour), tile_id) - results = append(results, id) - } - - return results -} - -func IncrementYear(t time.Time) time.Time { - return t.AddDate(1, 0, 0) -} -func IncrementMonth(t time.Time) time.Time { - return t.AddDate(0, 1, 0) -} -func IncrementDay(t time.Time) time.Time { - return t.AddDate(0, 0, 1) -} -func IncrementHour(t time.Time) time.Time { - return t.Add(time.Hour) -} - -func sameYear(t1, t2 time.Time) bool { - y1, _, _ := t1.Date() - y2, _, _ := t2.Date() - return (y1 == y2) -} - -func NextYear(start time.Time, end time.Time) bool { - nextYear := start.AddDate(1, 0, 0) - return sameYear(nextYear, end) || end.After(nextYear) -} - -func sameMonth(t1, t2 time.Time) bool { - y1, m1, _ := t1.Date() - y2, m2, _ := t2.Date() - return (y1 == y2) && (m1 == m2) -} - -func NextMonth(start time.Time, end time.Time) bool { - nextMonth := start.AddDate(0, 1, 0) - return sameMonth(nextMonth, end) || end.After(nextMonth) -} - -func sameDay(t1, t2 time.Time) bool { - y1, m1, d1 := t1.Date() - y2, m2, d2 := t2.Date() - return (y1 == y2) && (m1 == m2) && (d1 == d2) -} - -func NextDay(start time.Time, end time.Time) bool { - nextDay := start.AddDate(0, 0, 1) - return sameDay(nextDay, end) || end.After(nextDay) -} - -func GetRange(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 { - results, marker := upHill(start_time, end_time, tile_id) - r2 := downHill(marker, end_time, tile_id) - results = append(results, r2...) - return results -} - -func upHill(start_time time.Time, end_time time.Time, tile_id uint64) ([]uint64, time.Time) { - var results []uint64 - time_iterator := start_time - for time_iterator.Before(end_time) { - if NextDay(time_iterator, end_time) { - if time_iterator.Hour() == 0 { - if NextMonth(time_iterator, end_time) { - if time_iterator.Day() == 1 { - if NextYear(time_iterator, end_time) { - if time_iterator.Month() == 1 { - break - } else { - - month_id := GetTimeID(YM, uint(time_iterator.Year()), uint(time_iterator.Month()), 0, 0, tile_id) - results = append(results, month_id) - time_iterator = IncrementMonth(time_iterator) - } - } else { - break - } - } else { - day_id := GetTimeID(YMD, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), 0, tile_id) - results = append(results, day_id) - time_iterator = IncrementDay(time_iterator) - } - } else { - break - } - } else { - hour_id := GetTimeID(YMDH, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), uint(time_iterator.Hour()), tile_id) - results = append(results, hour_id) - time_iterator = IncrementHour(time_iterator) - } - } else { - break - } - } - return results, time_iterator - -} - -func downHill(start_time time.Time, end_time time.Time, tile_id uint64) []uint64 { - var results []uint64 - time_iterator := start_time - for time_iterator.Before(end_time) { - if NextYear(time_iterator, end_time) { - year_id := GetTimeID(Y, uint(time_iterator.Year()), 0, 0, 0, tile_id) - results = append(results, year_id) - time_iterator = IncrementYear(time_iterator) - } else if NextMonth(time_iterator, end_time) { - month_id := GetTimeID(YM, uint(time_iterator.Year()), uint(time_iterator.Month()), 0, 0, tile_id) - results = append(results, month_id) - time_iterator = IncrementMonth(time_iterator) - } else if NextDay(time_iterator, end_time) { - day_id := GetTimeID(YMD, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), 0, tile_id) - results = append(results, day_id) - time_iterator = IncrementDay(time_iterator) - } else { - hour_id := GetTimeID(YMDH, uint(time_iterator.Year()), uint(time_iterator.Month()), uint(time_iterator.Day()), uint(time_iterator.Hour()), tile_id) - results = append(results, hour_id) - time_iterator = IncrementHour(time_iterator) - } - } - return results -} diff --git a/timeframe_test.go b/timeframe_test.go deleted file mode 100644 index f31fcc32e..000000000 --- a/timeframe_test.go +++ /dev/null @@ -1,134 +0,0 @@ -package pilosa_test - -import ( - "testing" - "time" - - "github.com/umbel/pilosa" -) - -func TestGetRange_1h_0(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-08-11 14:00"), - MustParseTime("2014-08-11 16:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h_1(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 10:03"), - MustParseTime("2014-01-02 11:03"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_2h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 10:03"), - MustParseTime("2014-01-02 12:03"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_24h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 12:03"), - MustParseTime("2014-01-03 12:03"), - uint64(1), - ); len(m) != 24 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1d(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 00:00"), - MustParseTime("2014-01-03 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1d1h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 00:00"), - MustParseTime("2014-01-03 01:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h1d(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 23:00"), - MustParseTime("2014-01-04 00:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h1d1h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-02 23:00"), - MustParseTime("2014-01-04 01:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1y(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-01 00:00"), - MustParseTime("2015-01-01 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h1d1m(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-30 23:00"), - MustParseTime("2014-03-01 00:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetRange_1h1d1m1d1h(t *testing.T) { - if m := pilosa.GetRange( - MustParseTime("2014-01-30 23:00"), - MustParseTime("2014-03-02 01:00"), - uint64(1), - ); len(m) != 5 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestGetTimeIds(t *testing.T) { - _ = pilosa.GetTimeIds(uint64(15027), MustParseTime("1970-01-01 00:00"), pilosa.YMD) -} - -// DefaultTimeLayout is the time layout used by the tests. -const DefaultTimeLayout = "2006-01-02 15:04" - -// MustParseTime parses value using DefaultTimeLayout. Panic on error. -func MustParseTime(value string) time.Time { - v, err := time.Parse(DefaultTimeLayout, value) - if err != nil { - panic(err) - } - return v -}