From c6e2b1294b3e2cac05d6f791b2e4e00eb4b2708a Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 8 Dec 2016 12:29:49 -0700 Subject: [PATCH] Add time-based frames and Range() support. --- attr.go | 40 +- bitmap.go | 4 +- cache.go | 9 +- client.go | 24 +- client_test.go | 6 +- db.go | 149 ++ db_test.go | 68 + executor.go | 52 +- executor_test.go | 72 +- fragment.go | 44 +- fragment_test.go | 46 +- frame.go | 97 +- frame_test.go | 35 + handler.go | 184 +- handler_test.go | 101 +- index_test.go | 44 +- internal/internal.go | 2 + internal/internal.pb.go | 3847 +++++++++++++++++++++++++++++++++++---- internal/internal.proto | 84 +- pilosa.go | 13 +- server.go | 3 +- time.go | 228 +-- time_test.go | 242 +-- 23 files changed, 4561 insertions(+), 833 deletions(-) diff --git a/attr.go b/attr.go index dacbf9bbb..1425676cf 100644 --- a/attr.go +++ b/attr.go @@ -17,6 +17,13 @@ import ( // AttrBlockSize is the size of attribute blocks for anti-entropy. const AttrBlockSize = 100 +// Attribute data type enum. +const ( + AttrTypeString = 1 + AttrTypeUint = 2 + AttrTypeBool = 3 +) + // AttrStore represents a storage layer for attributes. type AttrStore struct { mu sync.Mutex @@ -303,32 +310,39 @@ func decodeAttrs(pb []*internal.Attr) map[string]interface{} { // encodeAttr converts a key/value pair into an Attr internal representation. func encodeAttr(key string, value interface{}) *internal.Attr { - pb := &internal.Attr{Key: proto.String(key)} + pb := &internal.Attr{Key: key} switch value := value.(type) { case string: - pb.StringValue = proto.String(value) + pb.Type = AttrTypeString + pb.StringValue = value case float64: - pb.UintValue = proto.Uint64(uint64(value)) + pb.Type = AttrTypeUint + pb.UintValue = uint64(value) case uint64: - pb.UintValue = proto.Uint64(value) + pb.Type = AttrTypeUint + pb.UintValue = value case int64: - pb.UintValue = proto.Uint64(uint64(value)) + pb.Type = AttrTypeUint + pb.UintValue = uint64(value) case bool: - pb.BoolValue = proto.Bool(value) + pb.Type = AttrTypeBool + pb.BoolValue = value } return pb } // decodeAttr converts from an Attr internal representation to a key/value pair. func decodeAttr(attr *internal.Attr) (key string, value interface{}) { - if attr.StringValue != nil { - return attr.GetKey(), attr.GetStringValue() - } else if attr.UintValue != nil { - return attr.GetKey(), attr.GetUintValue() - } else if attr.BoolValue != nil { - return attr.GetKey(), attr.GetBoolValue() + switch attr.Type { + case AttrTypeString: + return attr.Key, attr.StringValue + case AttrTypeUint: + return attr.Key, attr.UintValue + case AttrTypeBool: + return attr.Key, attr.BoolValue + default: + return attr.Key, nil } - return attr.GetKey(), nil } // cloneAttrs returns a shallow clone of m. diff --git a/bitmap.go b/bitmap.go index b3d8d0096..55664b24b 100644 --- a/bitmap.go +++ b/bitmap.go @@ -227,8 +227,8 @@ func decodeBitmap(pb *internal.Bitmap) *Bitmap { } b := NewBitmap() - b.Attrs = decodeAttrs(pb.GetAttrs()) - for _, v := range pb.GetBits() { + b.Attrs = decodeAttrs(pb.Attrs) + for _, v := range pb.Bits { b.SetBit(v) } return b diff --git a/cache.go b/cache.go index 4efce34f0..a1cba3dec 100644 --- a/cache.go +++ b/cache.go @@ -7,7 +7,6 @@ import ( "sort" "time" - "github.com/gogo/protobuf/proto" "github.com/golang/groupcache/lru" "github.com/pilosa/pilosa/internal" ) @@ -216,15 +215,15 @@ type Pair struct { func encodePair(p Pair) *internal.Pair { return &internal.Pair{ - Key: proto.Uint64(p.Key), - Count: proto.Uint64(p.Count), + Key: p.Key, + Count: p.Count, } } func decodePair(pb *internal.Pair) Pair { return Pair{ - Key: pb.GetKey(), - Count: pb.GetCount(), + Key: pb.Key, + Count: pb.Count, } } diff --git a/client.go b/client.go index d9796700d..9c13fe2ff 100644 --- a/client.go +++ b/client.go @@ -151,9 +151,9 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire // Encode query request. buf, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String(db), - Query: proto.String(query), - Remote: proto.Bool(!allowRedirect), + DB: db, + Query: query, + Remote: !allowRedirect, }) if err != nil { return nil, fmt.Errorf("marshal: %s", err) @@ -187,7 +187,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire var qresp internal.QueryResponse if err := proto.Unmarshal(body, &qresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.GetErr(); s != "" { + } else if s := qresp.Err; s != "" { return nil, errors.New(s) } @@ -230,9 +230,9 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e // Marshal bits to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ - DB: proto.String(db), - Frame: proto.String(frame), - Slice: proto.Uint64(slice), + DB: db, + Frame: frame, + Slice: slice, BitmapIDs: bitmapIDs, ProfileIDs: profileIDs, }) @@ -272,7 +272,7 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { var isresp internal.ImportResponse if err := proto.Unmarshal(body, &isresp); err != nil { return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.GetErr(); s != "" { + } else if s := isresp.Err; s != "" { return errors.New(s) } @@ -644,10 +644,10 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uin // BlockData returns bitmap/profile id pairs for a block. func (c *Client) BlockData(ctx context.Context, db, frame string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ - DB: proto.String(db), - Frame: proto.String(frame), - Slice: proto.Uint64(slice), - Block: proto.Uint64(uint64(block)), + DB: db, + Frame: frame, + Slice: slice, + Block: uint64(block), }) if err != nil { return nil, nil, err diff --git a/client_test.go b/client_test.go index d81e42a72..f37fea6ee 100644 --- a/client_test.go +++ b/client_test.go @@ -96,11 +96,11 @@ func TestClient_FragmentBlocks(t *testing.T) { defer idx.Close() // Set two bits on blocks 0 & 3. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(pilosa.HashBlockSize*3, 100, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, 1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, 1) s := NewServer() defer s.Close() diff --git a/db.go b/db.go index a28ec9cf7..f4e88333a 100644 --- a/db.go +++ b/db.go @@ -3,10 +3,15 @@ package pilosa import ( "errors" "fmt" + "io/ioutil" "os" "path/filepath" "sort" "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) // DB represents a container for frames. @@ -15,6 +20,10 @@ type DB struct { path string name string + // Default time quantum for all frames in database. + // This can be overridden by individual frames. + timeQuantum TimeQuantum + // Frames by name. frames map[string]*Frame @@ -53,6 +62,11 @@ func (db *DB) Open() error { return err } + // Read meta file. + if err := db.loadMeta(); err != nil { + return err + } + if err := db.openFrames(); err != nil { return err } @@ -93,6 +107,45 @@ func (db *DB) openFrames() error { return nil } +// loadMeta reads meta data for the database, if any. +func (db *DB) loadMeta() error { + var pb internal.DB + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(db.path, "meta")) + if os.IsNotExist(err) { + db.timeQuantum = "" + return nil + } else if err != nil { + return err + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + } + + // Copy metadata fields. + db.timeQuantum = TimeQuantum(pb.TimeQuantum) + + return nil +} + +// saveMeta writes meta data for the database. +func (db *DB) saveMeta() error { + // Marshal metadata. + buf, err := proto.Marshal(&internal.DB{TimeQuantum: string(db.timeQuantum)}) + if err != nil { + return err + } + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(db.path, "meta"), buf, 0666); err != nil { + return err + } + + return nil +} + // Close closes the database and its frames. func (db *DB) Close() error { db.mu.Lock() @@ -126,6 +179,34 @@ func (db *DB) SliceN() uint64 { return max } +// TimeQuantum returns the default time quantum for the database. +func (db *DB) TimeQuantum() TimeQuantum { + db.mu.Lock() + defer db.mu.Unlock() + return db.timeQuantum +} + +// SetTimeQuantum sets the default time quantum for the database. +func (db *DB) SetTimeQuantum(q TimeQuantum) error { + db.mu.Lock() + defer db.mu.Unlock() + + // Validate input. + if !q.Valid() { + return ErrInvalidTimeQuantum + } + + // Update value on database. + db.timeQuantum = q + + // Perist meta data to disk. + if err := db.saveMeta(); err != nil { + return err + } + + return nil +} + // FramePath returns the path to a frame in the database. func (db *DB) FramePath(name string) string { return filepath.Join(db.path, name) } @@ -187,6 +268,74 @@ func (db *DB) newFrame(path, name string) *Frame { return f } +// DeleteFrame removes a frame from the database. +func (db *DB) DeleteFrame(name string) error { + db.mu.Lock() + defer db.mu.Unlock() + + // Ignore if frame doesn't exist. + f := db.frame(name) + if f == nil { + return nil + } + + // Close frame. + if err := f.Close(); err != nil { + return err + } + + // Delete frame directory. + if err := os.RemoveAll(db.FramePath(name)); err != nil { + return err + } + + // Remove reference. + delete(db.frames, name) + + db.stats.Count("frameN", -1) + + return nil +} + +// SetBit sets a bit for a given profile & bitmap. +// If a timestamp is specified then set all bits for the different quantum units. +func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) { + // Read frame. + f, err := db.CreateFrameIfNotExists(name) + if err != nil { + return changed, err + } + + // If this is a non-time bit then simply set the bit on the frame. + if t == nil { + return f.SetBit(bitmapID, profileID) + } + + // Determine quantum of frame. Set to the default quantum if it is unset. + q := f.TimeQuantum() + if q == "" { + q = db.TimeQuantum() + if err := f.SetTimeQuantum(q); err != nil { + return changed, err + } + } + + // If a timestamp is specified then set bits across all frames for the quantum. + for _, subname := range FramesByTime(name, *t, q) { + f, err := db.CreateFrameIfNotExists(subname) + if err != nil { + return changed, err + } + + if c, err := f.SetBit(bitmapID, profileID); err != nil { + return changed, err + } else if c { + changed = true + } + } + return changed, nil +} + type dbSlice []*DB func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/db_test.go b/db_test.go index 5ffc68392..46a138c87 100644 --- a/db_test.go +++ b/db_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "os" "testing" + "time" "github.com/pilosa/pilosa" ) @@ -34,6 +35,49 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) { } } +// Ensure database can delete a frame. +func TestDB_DeleteFrame(t *testing.T) { + db := MustOpenDB() + defer db.Close() + + // Create frame. + if _, err := db.CreateFrameIfNotExists("f"); err != nil { + t.Fatal(err) + } + + // Delete frame & verify it's gone. + if err := db.DeleteFrame("f"); err != nil { + t.Fatal(err) + } else if db.Frame("f") != nil { + t.Fatal("expected nil frame") + } + + // Delete again to make sure it doesn't error. + if err := db.DeleteFrame("f"); err != nil { + t.Fatal(err) + } +} + +// Ensure database can set the default time quantum. +func TestDB_SetTimeQuantum(t *testing.T) { + db := MustOpenDB() + defer db.Close() + + // Set & retrieve time quantum. + if err := db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload database and verify that it is persisted. + if err := db.Reopen(); err != nil { + t.Fatal(err) + } else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} + // DB represents a test wrapper for pilosa.DB. type DB struct { *pilosa.DB @@ -63,3 +107,27 @@ func (db *DB) Close() error { defer os.RemoveAll(db.Path()) return db.DB.Close() } + +// Reopen closes the database and reopens it. +func (db *DB) Reopen() error { + if err := db.DB.Close(); err != nil { + return err + } + + path, name := db.Path(), db.Name() + db.DB = pilosa.NewDB(path, name) + + if err := db.Open(); err != nil { + return err + } + return nil +} + +// MustSetBit sets a bit on the database. Panic on error. +func (db *DB) MustSetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool) { + changed, err := db.SetBit(name, bitmapID, profileID, t) + if err != nil { + panic(err) + } + return changed +} diff --git a/executor.go b/executor.go index abf2d750e..bd0f16aba 100644 --- a/executor.go +++ b/executor.go @@ -304,11 +304,28 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Rang frame = DefaultFrame } - f := e.Index.Fragment(db, frame, slice) + // Retrieve base frame. + f := e.Index.Frame(db, frame) if f == nil { - return NewBitmap(), nil + return &Bitmap{}, nil } - return f.Range(c.ID, c.StartTime, c.EndTime), nil + + // If no quantum exists then return an empty bitmap. + q := f.TimeQuantum() + if q == "" { + return &Bitmap{}, nil + } + + // Union bitmaps across all time-based subframes. + bm := &Bitmap{} + for _, subframe := range FramesByTimeRange(frame, c.StartTime, c.EndTime, q) { + f := e.Index.Fragment(db, subframe, slice) + if f == nil { + continue + } + bm = bm.Union(f.Bitmap(c.ID)) + } + return bm, nil } // executeUnionSlice executes a union() call for a local slice. @@ -405,15 +422,14 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit, for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - f, err := e.Index.CreateFragmentIfNotExists(db, c.Frame, slice) + db, err := e.Index.CreateDBIfNotExists(db) if err != nil { - return false, fmt.Errorf("fragment: %s", err) + return false, fmt.Errorf("db: %s", err) } - val, err := f.SetBit(c.ID, c.ProfileID, opt.Timestamp, opt.Quantum) + val, err := db.SetBit(c.Frame, c.ID, c.ProfileID, opt.Timestamp) if err != nil { return false, err - } - if val { + } else if val { ret = true } continue @@ -579,14 +595,13 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { // Encode request object. pbreq := &internal.QueryRequest{ - DB: proto.String(db), - Query: proto.String(q.String()), - Slices: slices, - Quantum: proto.Uint32(uint32(opt.Quantum)), - Remote: proto.Bool(true), + DB: db, + Query: q.String(), + Slices: slices, + Remote: true, } if opt.Timestamp != nil { - pbreq.Timestamp = proto.Int64(opt.Timestamp.UnixNano()) + pbreq.Timestamp = opt.Timestamp.UnixNano() } buf, err := proto.Marshal(pbreq) if err != nil { @@ -632,7 +647,7 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query } // Return an error, if specified on response. - if err := decodeError(pb.GetErr()); err != nil { + if err := decodeError(pb.Err); err != nil { return nil, err } @@ -648,11 +663,11 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query case *pql.TopN: v, err = decodePairs(pb.Results[i].GetPairs()), nil case *pql.Count: - v, err = pb.Results[i].GetN(), nil + v, err = pb.Results[i].N, nil case *pql.SetBit: - v, err = pb.Results[i].GetChanged(), nil + v, err = pb.Results[i].Changed, nil case *pql.ClearBit: - v, err = pb.Results[i].GetChanged(), nil + v, err = pb.Results[i].Changed, nil case *pql.SetBitmapAttrs: case *pql.SetProfileAttrs: default: @@ -837,7 +852,6 @@ type mapResponse struct { // ExecOptions represents an execution context for a single Execute() call. type ExecOptions struct { Timestamp *time.Time - Quantum TimeQuantum Remote bool } diff --git a/executor_test.go b/executor_test.go index 999c841d0..c0f498dad 100644 --- a/executor_test.go +++ b/executor_test.go @@ -171,15 +171,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -197,12 +197,12 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 2, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth+2, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 2) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -221,19 +221,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2) // Create an intersecting bitmap. - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -253,15 +253,23 @@ func TestExecutor_Execute_Range(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - f := idx.MustCreateFragmentIfNotExists("d", "f.t", 0) - if _, err := f.SetBit(1, 100, MustParseTime("2000-01-01 00:00"), pilosa.YMD); err != nil { - t.Fatal(err) - } + db := idx.MustCreateDBIfNotExists("d") + db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")) + db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-31 00:00")) + db.MustSetBit("f", 1, 3, MustParseTimePtr("2000-01-01 00:00")) + db.MustSetBit("f", 1, 4, MustParseTimePtr("2000-01-02 00:00")) + db.MustSetBit("f", 1, 5, MustParseTimePtr("2000-02-01 00:00")) + db.MustSetBit("f", 1, 6, MustParseTimePtr("2001-01-01 00:00")) + db.MustSetBit("f", 1, 7, MustParseTimePtr("2002-01-01 02:00")) + + db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early + db.MustSetBit("f", 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late + db.MustSetBit("f", 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f.t, start="2000-01-01T00:00", end="2000-01-01T01:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) { + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected bits: %+v", bits) } } diff --git a/fragment.go b/fragment.go index d0bb2e73e..3eea7cf03 100644 --- a/fragment.go +++ b/fragment.go @@ -238,7 +238,7 @@ func (f *Fragment) openCache() error { // Read in all bitmaps by ID. // This will cause them to be added to the cache. - for _, bitmapID := range pb.GetBitmapIDs() { + for _, bitmapID := range pb.BitmapIDs { n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) f.cache.Add(bitmapID, n) } @@ -323,11 +323,6 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { } bm.InvalidateCount() - // f.storage.ForEachRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth, func(i uint64) { - // profileID := (f.slice * SliceWidth) + (i % SliceWidth) - // bm.SetBit(profileID) - // }) - // Update cache. f.cache.Add(bitmapID, bm.Count()) @@ -336,15 +331,9 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { // SetBit sets a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(bitmapID, profileID uint64, t *time.Time, q TimeQuantum) (changed bool, err error) { +func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - - // Set time bits if this is a time-frame and a timestamp is specified. - if strings.HasSuffix(f.frame, FrameSuffixTime) && t != nil { - return f.setTimeBit(bitmapID, profileID, *t, q) - } - return f.setBit(bitmapID, profileID) } @@ -379,17 +368,6 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) return changed, nil } -func (f *Fragment) setTimeBit(bitmapID, profileID uint64, t time.Time, q TimeQuantum) (changed bool, err error) { - for _, timeID := range TimeIDsFromQuantum(q, t, bitmapID) { - if v, err := f.setBit(timeID, profileID); err != nil { - return changed, fmt.Errorf("set time bit: t=%s, q=%s, err=%s", t, q, err) - } else if v { - changed = true - } - } - return changed, nil -} - // ClearBit clears a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) { @@ -601,24 +579,6 @@ type TopOptions struct { FilterValues []interface{} } -func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap { - f.mu.Lock() - defer f.mu.Unlock() - - // Retrieve a list of bitmap ids for a given time range. - bitmapIDs := TimeIDsFromRange(start, end, bitmapID) - if len(bitmapIDs) == 0 { - return NewBitmap() - } - - // Union all bitmap ids from the time range. - bm := f.bitmap(bitmapIDs[0]) - for _, id := range bitmapIDs[1:] { - bm = bm.Union(f.bitmap(id)) - } - return bm -} - // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. func (f *Fragment) Checksum() []byte { diff --git a/fragment_test.go b/fragment_test.go index 11ce52dca..da2edac6c 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -27,11 +27,11 @@ func TestFragment_SetBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(120, 1, nil, 0); err != nil { + if _, err := f.SetBit(120, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 6, nil, 0); err != nil { + } else if _, err := f.SetBit(120, 6); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(121, 0, nil, 0); err != nil { + } else if _, err := f.SetBit(121, 0); err != nil { t.Fatal(err) } @@ -58,9 +58,9 @@ func TestFragment_ClearBit(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1, nil, 0); err != nil { + if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { + } else if _, err := f.SetBit(1000, 2); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -85,9 +85,9 @@ func TestFragment_Snapshot(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1, nil, 0); err != nil { + if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { + } else if _, err := f.SetBit(1000, 2); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -114,11 +114,11 @@ func TestFragment_ForEachBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(100, 20, nil, 0); err != nil { + if _, err := f.SetBit(100, 20); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 38, nil, 0); err != nil { + } else if _, err := f.SetBit(2, 38); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 37, nil, 0); err != nil { + } else if _, err := f.SetBit(2, 37); err != nil { t.Fatal(err) } @@ -284,9 +284,9 @@ func TestFragment_Checksum(t *testing.T) { // Retrieve checksum and set bits. orig := f.Checksum() - if _, err := f.SetBit(1, 200, nil, 0); err != nil { + if _, err := f.SetBit(1, 200); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.HashBlockSize*2, 200, nil, 0); err != nil { + } else if _, err := f.SetBit(pilosa.HashBlockSize*2, 200); err != nil { t.Fatal(err) } @@ -305,7 +305,7 @@ func TestFragment_Blocks(t *testing.T) { var prev []pilosa.FragmentBlock // Set first bit. - if _, err := f.SetBit(0, 0, nil, 0); err != nil { + if _, err := f.SetBit(0, 0); err != nil { t.Fatal(err) } blocks := f.Blocks() @@ -315,7 +315,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different bitmap. - if _, err := f.SetBit(20, 0, nil, 0); err != nil { + if _, err := f.SetBit(20, 0); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -325,7 +325,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different profile. - if _, err := f.SetBit(20, 100, nil, 0); err != nil { + if _, err := f.SetBit(20, 100); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -340,7 +340,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { defer f.Close() // Set bits on a different block. - if _, err := f.SetBit(100, 1, nil, 0); err != nil { + if _, err := f.SetBit(100, 1); err != nil { t.Fatal(err) } @@ -359,7 +359,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0, nil, 0); err != nil { + if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) } } @@ -391,7 +391,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0, nil, 0); err != nil { + if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) } } @@ -422,9 +422,9 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { defer f0.Close() // Set and then clear bits on the fragment. - if _, err := f0.SetBit(1000, 1, nil, 0); err != nil { + if _, err := f0.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f0.SetBit(1000, 2, nil, 0); err != nil { + } else if _, err := f0.SetBit(1000, 2); err != nil { t.Fatal(err) } else if _, err := f0.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -527,12 +527,12 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Generate some intersecting data. for i := 0; i < 10000; i += 2 { - if _, err := f.SetBit(1, uint64(i), nil, 0); err != nil { + if _, err := f.SetBit(1, uint64(i)); err != nil { b.Fatal(err) } } for i := 0; i < 10000; i += 3 { - if _, err := f.SetBit(2, uint64(i), nil, 0); err != nil { + if _, err := f.SetBit(2, uint64(i)); err != nil { b.Fatal(err) } } @@ -609,7 +609,7 @@ func (f *Fragment) Reopen() error { // This function does not accept a timestamp or quantum. func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { for _, profileID := range profileIDs { - if _, err := f.SetBit(bitmapID, profileID, nil, 0); err != nil { + if _, err := f.SetBit(bitmapID, profileID); err != nil { panic(err) } } diff --git a/frame.go b/frame.go index 56e4b3f1a..a3e90ce1e 100644 --- a/frame.go +++ b/frame.go @@ -2,26 +2,28 @@ package pilosa import ( "fmt" + "io/ioutil" "os" "path/filepath" "strconv" "sync" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) const ( - // FrameSuffixTime is the suffix used for time-based frames. - FrameSuffixTime = ".t" - // FrameSuffixRank is the suffix used for rank-based frames. FrameSuffixRank = ".n" ) // Frame represents a container for fragments. type Frame struct { - mu sync.Mutex - path string - db string - name string + mu sync.Mutex + path string + db string + name string + timeQuantum TimeQuantum // Fragments by slice. fragments map[uint64]*Fragment @@ -80,6 +82,10 @@ func (f *Frame) Open() error { return err } + if err := f.loadMeta(); err != nil { + return err + } + if err := f.openFragments(); err != nil { return err } @@ -134,6 +140,45 @@ func (f *Frame) openFragments() error { return nil } +// loadMeta reads meta data for the frame, if any. +func (f *Frame) loadMeta() error { + var pb internal.Frame + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(f.path, "meta")) + if os.IsNotExist(err) { + f.timeQuantum = "" + return nil + } else if err != nil { + return err + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + } + + // Copy metadata fields. + f.timeQuantum = TimeQuantum(pb.TimeQuantum) + + return nil +} + +// saveMeta writes meta data for the frame. +func (f *Frame) saveMeta() error { + // Marshal metadata. + buf, err := proto.Marshal(&internal.Frame{TimeQuantum: string(f.timeQuantum)}) + if err != nil { + return err + } + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(f.path, "meta"), buf, 0666); err != nil { + return err + } + + return nil +} + // Close closes the frame and its fragments. func (f *Frame) Close() error { f.mu.Lock() @@ -153,6 +198,34 @@ func (f *Frame) Close() error { return nil } +// TimeQuantum returns the time quantum for the frame. +func (f *Frame) TimeQuantum() TimeQuantum { + f.mu.Lock() + defer f.mu.Unlock() + return f.timeQuantum +} + +// SetTimeQuantum sets the time quantum for the frame. +func (f *Frame) SetTimeQuantum(q TimeQuantum) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Validate input. + if !q.Valid() { + return ErrInvalidTimeQuantum + } + + // Update value on frame. + f.timeQuantum = q + + // Perist meta data to disk. + if err := f.saveMeta(); err != nil { + return err + } + + return nil +} + // FragmentPath returns the path to a fragment in the frame. func (f *Frame) FragmentPath(slice uint64) string { return filepath.Join(f.path, strconv.FormatUint(slice, 10)) @@ -213,6 +286,16 @@ func (f *Frame) newFragment(path string, slice uint64) *Fragment { return frag } +// SetBit sets a bit within the frame. +func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) { + slice := bitmapID / SliceWidth + frag, err := f.CreateFragmentIfNotExists(slice) + if err != nil { + return changed, err + } + return frag.SetBit(bitmapID, profileID) +} + type frameSlice []*Frame func (p frameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/frame_test.go b/frame_test.go index 01d508ef9..186db0c62 100644 --- a/frame_test.go +++ b/frame_test.go @@ -34,6 +34,26 @@ func TestFrame_CreateFragmentIfNotExists(t *testing.T) { } } +// Ensure frame can set its time quantum. +func TestFrame_SetTimeQuantum(t *testing.T) { + f := MustOpenFrame() + defer f.Close() + + // Set & retrieve time quantum. + if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload frame and verify that it is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} + // Frame represents a test wrapper for pilosa.Frame. type Frame struct { *pilosa.Frame @@ -63,3 +83,18 @@ func (f *Frame) Close() error { defer os.RemoveAll(f.Path()) return f.Frame.Close() } + +// Reopen closes the database and reopens it. +func (f *Frame) Reopen() error { + if err := f.Frame.Close(); err != nil { + return err + } + + path, db, name := f.Path(), f.DB(), f.Name() + f.Frame = pilosa.NewFrame(path, db, name) + + if err := f.Open(); err != nil { + return err + } + return nil +} diff --git a/handler.go b/handler.go index 9899133db..f9088deb1 100644 --- a/handler.go +++ b/handler.go @@ -114,6 +114,13 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } + case "/db/time_quantum": + switch r.Method { + case "PATCH": + h.handlePatchDBTimeQuantum(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } case "/db/attr/diff": switch r.Method { case "POST": @@ -121,6 +128,20 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } + case "/frame": + switch r.Method { + case "DELETE": + h.handleDeleteFrame(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } + case "/frame/time_quantum": + switch r.Method { + case "PATCH": + h.handlePatchFrameTimeQuantum(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } case "/frame/attr/diff": switch r.Method { case "POST": @@ -203,7 +224,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // Build execution options. opt := &ExecOptions{ Timestamp: req.Timestamp, - Quantum: req.Quantum, Remote: req.Remote, } @@ -256,7 +276,7 @@ func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) erro sm := h.Index.SliceN() if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { pb := &internal.SliceMaxResponse{ - SliceMax: &sm, + SliceMax: sm, } if buf, err := proto.Marshal(pb); err != nil { return err @@ -299,6 +319,48 @@ type deleteDBRequest struct { type deleteDBResponse struct{} +// handlePatchDBTimeQuantum handles PATCH /db/time_quantum request. +func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req patchDBTimeQuantumRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Validate quantum. + tq, err := ParseTimeQuantum(req.TimeQuantum) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Retrieve database by name. + db, err := h.Index.CreateDBIfNotExists(req.DB) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Set default time quantum on database. + if err := db.SetTimeQuantum(tq); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(patchDBTimeQuantumResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type patchDBTimeQuantumRequest struct { + DB string `json:"db"` + TimeQuantum string `json:"time_quantum"` +} + +type patchDBTimeQuantumResponse struct{} + // handlePostDBAttrDiff handles POST /db/attr/diff requests. func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -355,6 +417,86 @@ type postDBAttrDiffResponse struct { Attrs map[uint64]map[string]interface{} `json:"attrs"` } +// handleDeleteFrame handles DELETE /frame request. +func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req deleteFrameRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Find database. + db := h.Index.DB(req.DB) + if db == nil { + if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } + return + } + + // Delete frame from the database. + if err := db.DeleteFrame(req.Frame); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type deleteFrameRequest struct { + DB string `json:"db"` + Frame string `json:"frame"` +} + +type deleteFrameResponse struct{} + +// handlePatchFrameTimeQuantum handles PATCH /frame/time_quantum request. +func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req patchFrameTimeQuantumRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Validate quantum. + tq, err := ParseTimeQuantum(req.TimeQuantum) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Retrieve database by name. + f, err := h.Index.CreateFrameIfNotExists(req.DB, req.Frame) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Set default time quantum on database. + if err := f.SetTimeQuantum(tq); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type patchFrameTimeQuantumRequest struct { + DB string `json:"db"` + Frame string `json:"frame"` + TimeQuantum string `json:"time_quantum"` +} + +type patchFrameTimeQuantumResponse struct{} + // handlePostFrameAttrDiff handles POST /frame/attr/diff requests. func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -495,7 +637,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { } // Parse time granularity. - quantum := YMDH + quantum := TimeQuantum("YMDH") if s := q.Get("time_granularity"); s != "" { v, err := ParseTimeQuantum(s) if err != nil { @@ -561,7 +703,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - db, frame, slice := req.GetDB(), req.GetFrame(), req.GetSlice() + db, frame, slice := req.DB, req.Frame, req.Slice // Validate that this handler owns the slice. if !h.Cluster.OwnsFragment(h.Host, db, slice) { @@ -578,16 +720,16 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { http.Error(w, "fragment error", http.StatusInternalServerError) return } - h.logger().Println("Import into Fragment:", db, frame, slice, len(req.GetProfileIDs())) + h.logger().Println("Import into Fragment:", db, frame, slice, len(req.ProfileIDs)) // Import into fragment. - err = f.Import(req.GetBitmapIDs(), req.GetProfileIDs()) + err = f.Import(req.BitmapIDs, req.ProfileIDs) if err != nil { - h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", db, frame, slice, len(req.GetProfileIDs()), err) + h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", db, frame, slice, len(req.ProfileIDs), err) } // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: proto.String(errorString(err))}) + buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) if e != nil { http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) return @@ -733,7 +875,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ } // Retrieve fragment from index. - f := h.Index.Fragment(req.GetDB(), req.GetFrame(), req.GetSlice()) + f := h.Index.Fragment(req.DB, req.Frame, req.Slice) if f == nil { http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) return @@ -742,7 +884,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ // Read data var resp internal.BlockDataResponse if f != nil { - resp.BitmapIDs, resp.ProfileIDs = f.BlockData(int(req.GetBlock())) + resp.BitmapIDs, resp.ProfileIDs = f.BlockData(int(req.Block)) } // Encode response. @@ -919,16 +1061,16 @@ type QueryRequest struct { func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { req := &QueryRequest{ - DB: pb.GetDB(), - Query: pb.GetQuery(), - Slices: pb.GetSlices(), - Profiles: pb.GetProfiles(), - Quantum: TimeQuantum(pb.GetQuantum()), - Remote: pb.GetRemote(), + DB: pb.DB, + Query: pb.Query, + Slices: pb.Slices, + Profiles: pb.Profiles, + Quantum: TimeQuantum(pb.Quantum), + Remote: pb.Remote, } - if pb.Timestamp != nil { - t := time.Unix(0, pb.GetTimestamp()) + if pb.Timestamp != 0 { + t := time.Unix(0, pb.Timestamp) req.Timestamp = &t } @@ -978,14 +1120,14 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { case []Pair: pb.Results[i].Pairs = encodePairs(result) case uint64: - pb.Results[i].N = proto.Uint64(result) + pb.Results[i].N = result case bool: - pb.Results[i].Changed = proto.Bool(result) + pb.Results[i].Changed = result } } if resp.Err != nil { - pb.Err = proto.String(resp.Err.Error()) + pb.Err = resp.Err.Error() } return pb diff --git a/handler_test.go b/handler_test.go index 90ecfa608..237133d78 100644 --- a/handler_test.go +++ b/handler_test.go @@ -93,8 +93,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Generate request body. reqBody, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String("db0"), - Query: proto.String("Count(Bitmap(100))"), + DB: "db0", + Query: "Count(Bitmap(100))", Slices: []uint64{0, 1}, }) if err != nil { @@ -157,7 +157,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if n := resp.Results[0].GetN(); n != 100 { + } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) } } @@ -232,15 +232,15 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { + } else if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) - } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { + } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].GetKey(), attrs[1].GetUintValue(); k != "c" || v != uint64(1) { + } else if k, v := attrs[1].Key, attrs[1].UintValue; k != "c" || v != uint64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].GetKey(), attrs[2].GetBoolValue(); k != "d" || v != true { + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v) } } @@ -268,9 +268,9 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Encode request body. buf, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String("d"), - Query: proto.String("Bitmap(100)"), - Profiles: proto.Bool(true), + DB: "d", + Query: "Bitmap(100)", + Profiles: true, }) if err != nil { t.Fatal(err) @@ -289,25 +289,25 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { + if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) - } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { + } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].GetKey(), attrs[1].GetUintValue(); k != "c" || v != uint64(1) { + } else if k, v := attrs[1].Key, attrs[1].UintValue; k != "c" || v != uint64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].GetKey(), attrs[2].GetBoolValue(); k != "d" || v != true { + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v) } - if a := resp.GetProfiles(); len(a) != 1 { + if a := resp.Profiles; len(a) != 1 { t.Fatalf("unexpected profiles length: %d", len(a)) - } else if a[0].GetID() != 1 { - t.Fatalf("unexpected id: %d", a[0].GetID()) - } else if len(a[0].GetAttrs()) != 1 { + } else if a[0].ID != 1 { + t.Fatalf("unexpected id: %d", a[0].ID) + } else if len(a[0].Attrs) != 1 { t.Fatalf("unexpected profile attr length: %d", len(a)) - } else if k, v := a[0].GetAttrs()[0].GetKey(), a[0].GetAttrs()[0].GetStringValue(); k != "x" || v != "y" { + } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) } } @@ -391,7 +391,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if s := resp.GetErr(); s != `marker` { + } else if s := resp.Err; s != `marker` { t.Fatalf("unexpected error: %s", s) } } @@ -453,6 +453,63 @@ func TestHandler_DB_Delete(t *testing.T) { } } +// Ensure handler can delete a frame. +func TestHandler_DeleteFrame(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + if _, err := idx.CreateFrameIfNotExists("d0", "f1"); err != nil { + t.Fatal(err) + } + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/frame", strings.NewReader(`{"db":"d0","frame":"f1"}`))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if f := idx.DB("d0").Frame("f1"); f != nil { + t.Fatal("expected nil frame") + } +} + +// Ensure handler can set the DB time quantum. +func TestHandler_SetDBTimeQuantum(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/time_quantum", strings.NewReader(`{"db":"d0","time_quantum":"ymdh"}`))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if q := idx.DB("d0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected time quantum: %s", q) + } +} + +// Ensure handler can set the frame time quantum. +func TestHandler_SetFrameTimeQuantum(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/frame/time_quantum", strings.NewReader(`{"db":"d0","frame":"f1","time_quantum":"ymdh"}`))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if q := idx.DB("d0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected time quantum: %s", q) + } +} + // Ensure the handler can return data in differing blocks for a database. func TestHandler_DB_AttrStore_Diff(t *testing.T) { idx := MustOpenIndex() diff --git a/index_test.go b/index_test.go index 851de772f..512c211c6 100644 --- a/index_test.go +++ b/index_test.go @@ -19,11 +19,11 @@ func TestIndex_DeleteDB(t *testing.T) { // Write bits to separate databases. f0 := idx.MustCreateFragmentIfNotExists("d0", "f", 0) - if _, err := f0.SetBit(100, 200, nil, 0); err != nil { + if _, err := f0.SetBit(100, 200); err != nil { t.Fatal(err) } f1 := idx.MustCreateFragmentIfNotExists("d1", "f", 0) - if _, err := f1.SetBit(100, 200, nil, 0); err != nil { + if _, err := f1.SetBit(100, 200); err != nil { t.Fatal(err) } @@ -74,18 +74,18 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Set data on the local index. f := idx0.MustCreateFragmentIfNotExists("d", "f", 0) - if _, err := f.SetBit(0, 10, nil, 0); err != nil { + if _, err := f.SetBit(0, 10); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 20, nil, 0); err != nil { + } else if _, err := f.SetBit(2, 20); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 10, nil, 0); err != nil { + } else if _, err := f.SetBit(120, 10); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(200, 4, nil, 0); err != nil { + } else if _, err := f.SetBit(200, 4); err != nil { t.Fatal(err) } f = idx0.MustCreateFragmentIfNotExists("d", "f0", 1) - if _, err := f.SetBit(9, SliceWidth+5, nil, 0); err != nil { + if _, err := f.SetBit(9, SliceWidth+5); err != nil { t.Fatal(err) } @@ -93,20 +93,20 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Set data on the remote index. f = idx1.MustCreateFragmentIfNotExists("d", "f", 0) - if _, err := f.SetBit(0, 4000, nil, 0); err != nil { + if _, err := f.SetBit(0, 4000); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(3, 10, nil, 0); err != nil { + } else if _, err := f.SetBit(3, 10); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 10, nil, 0); err != nil { + } else if _, err := f.SetBit(120, 10); err != nil { t.Fatal(err) } f = idx1.MustCreateFragmentIfNotExists("y", "z", 3) - if _, err := f.SetBit(10, (3*SliceWidth)+4, nil, 0); err != nil { + if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+5, nil, 0); err != nil { + } else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+7, nil, 0); err != nil { + } else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil { t.Fatal(err) } @@ -185,6 +185,24 @@ func (i *Index) Close() error { return i.Index.Close() } +// MustCreateDBIfNotExists returns a given db. Panic on error. +func (i *Index) MustCreateDBIfNotExists(db string) *DB { + d, err := i.Index.CreateDBIfNotExists(db) + if err != nil { + panic(err) + } + return &DB{DB: d} +} + +// MustCreateFrameIfNotExists returns a given frame. Panic on error. +func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame { + f, err := i.Index.CreateFrameIfNotExists(db, frame) + if err != nil { + panic(err) + } + return &Frame{Frame: f} +} + // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. func (i *Index) MustCreateFragmentIfNotExists(db, frame string, slice uint64) *Fragment { f, err := i.Index.CreateFragmentIfNotExists(db, frame, slice) diff --git a/internal/internal.go b/internal/internal.go index 574c0ee5c..3d322d82a 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -7,6 +7,8 @@ import ( "github.com/gogo/protobuf/proto" ) +//go:generate protoc --gofast_out=. internal.proto + type Request proto.Message type Response proto.Message diff --git a/internal/internal.pb.go b/internal/internal.pb.go index c61ad59c7..c177fdd24 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -1,36 +1,40 @@ // Code generated by protoc-gen-gogo. -// source: internal/internal.proto +// source: internal.proto // DO NOT EDIT! /* -Package internal is a generated protocol buffer package. + Package internal is a generated protocol buffer package. -It is generated from these files: - internal/internal.proto + It is generated from these files: + internal.proto -It has these top-level messages: - Bitmap - Pair - Bit - Profile - Attr - AttrMap - QueryRequest - QueryResponse - QueryResult - ImportRequest - ImportResponse - BlockDataRequest - BlockDataResponse - Cache - SliceMaxResponse + It has these top-level messages: + DB + Frame + Bitmap + Pair + Bit + Profile + Attr + AttrMap + QueryRequest + QueryResponse + QueryResult + ImportRequest + ImportResponse + BlockDataRequest + BlockDataResponse + Cache + SliceMaxResponse */ package internal -import proto "github.com/gogo/protobuf/proto" +import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import io "io" + // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf @@ -38,25 +42,37 @@ var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. -const _ = proto.GoGoProtoPackageIsVersion1 +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type DB struct { + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` +} + +func (m *DB) Reset() { *m = DB{} } +func (m *DB) String() string { return proto.CompactTextString(m) } +func (*DB) ProtoMessage() {} +func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } + +type Frame struct { + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` +} + +func (m *Frame) Reset() { *m = Frame{} } +func (m *Frame) String() string { return proto.CompactTextString(m) } +func (*Frame) ProtoMessage() {} +func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } type Bitmap struct { - Bits []uint64 `protobuf:"varint,1,rep,name=Bits" json:"Bits,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_unrecognized []byte `json:"-"` + Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Bitmap) Reset() { *m = Bitmap{} } func (m *Bitmap) String() string { return proto.CompactTextString(m) } func (*Bitmap) ProtoMessage() {} -func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } - -func (m *Bitmap) GetBits() []uint64 { - if m != nil { - return m.Bits - } - return nil -} +func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} } func (m *Bitmap) GetAttrs() []*Attr { if m != nil { @@ -66,72 +82,34 @@ func (m *Bitmap) GetAttrs() []*Attr { } type Pair struct { - Key *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"` - Count *uint64 `protobuf:"varint,2,req,name=Count" json:"Count,omitempty"` - XXX_unrecognized []byte `json:"-"` + Key uint64 `protobuf:"varint,1,opt,name=Key,proto3" json:"Key,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} -func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } - -func (m *Pair) GetKey() uint64 { - if m != nil && m.Key != nil { - return *m.Key - } - return 0 -} - -func (m *Pair) GetCount() uint64 { - if m != nil && m.Count != nil { - return *m.Count - } - return 0 -} +func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } type Bit struct { - BitmapID *uint64 `protobuf:"varint,1,req,name=BitmapID" json:"BitmapID,omitempty"` - ProfileID *uint64 `protobuf:"varint,2,req,name=ProfileID" json:"ProfileID,omitempty"` - XXX_unrecognized []byte `json:"-"` + BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,proto3" json:"BitmapID,omitempty"` + ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,proto3" json:"ProfileID,omitempty"` } func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} } - -func (m *Bit) GetBitmapID() uint64 { - if m != nil && m.BitmapID != nil { - return *m.BitmapID - } - return 0 -} - -func (m *Bit) GetProfileID() uint64 { - if m != nil && m.ProfileID != nil { - return *m.ProfileID - } - return 0 -} +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } type Profile struct { - ID *uint64 `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_unrecognized []byte `json:"-"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Profile) Reset() { *m = Profile{} } func (m *Profile) String() string { return proto.CompactTextString(m) } func (*Profile) ProtoMessage() {} -func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } - -func (m *Profile) GetID() uint64 { - if m != nil && m.ID != nil { - return *m.ID - } - return 0 -} +func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} } func (m *Profile) GetAttrs() []*Attr { if m != nil { @@ -141,55 +119,26 @@ func (m *Profile) GetAttrs() []*Attr { } type Attr struct { - Key *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"` - StringValue *string `protobuf:"bytes,2,opt,name=StringValue" json:"StringValue,omitempty"` - UintValue *uint64 `protobuf:"varint,3,opt,name=UintValue" json:"UintValue,omitempty"` - BoolValue *bool `protobuf:"varint,4,opt,name=BoolValue" json:"BoolValue,omitempty"` - XXX_unrecognized []byte `json:"-"` + Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` + Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` + StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` + UintValue uint64 `protobuf:"varint,4,opt,name=UintValue,proto3" json:"UintValue,omitempty"` + BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` } func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } - -func (m *Attr) GetKey() string { - if m != nil && m.Key != nil { - return *m.Key - } - return "" -} - -func (m *Attr) GetStringValue() string { - if m != nil && m.StringValue != nil { - return *m.StringValue - } - return "" -} - -func (m *Attr) GetUintValue() uint64 { - if m != nil && m.UintValue != nil { - return *m.UintValue - } - return 0 -} - -func (m *Attr) GetBoolValue() bool { - if m != nil && m.BoolValue != nil { - return *m.BoolValue - } - return false -} +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_unrecognized []byte `json:"-"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -199,88 +148,30 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Query *string `protobuf:"bytes,2,req,name=Query" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,3,rep,name=Slices" json:"Slices,omitempty"` - Profiles *bool `protobuf:"varint,4,opt,name=Profiles" json:"Profiles,omitempty"` - Timestamp *int64 `protobuf:"varint,5,opt,name=Timestamp" json:"Timestamp,omitempty"` - Quantum *uint32 `protobuf:"varint,6,opt,name=Quantum" json:"Quantum,omitempty"` - Remote *bool `protobuf:"varint,7,opt,name=Remote" json:"Remote,omitempty"` - XXX_unrecognized []byte `json:"-"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Query string `protobuf:"bytes,2,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,3,rep,packed,name=Slices" json:"Slices,omitempty"` + Profiles bool `protobuf:"varint,4,opt,name=Profiles,proto3" json:"Profiles,omitempty"` + Timestamp int64 `protobuf:"varint,5,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` + Quantum string `protobuf:"bytes,6,opt,name=Quantum,proto3" json:"Quantum,omitempty"` + Remote bool `protobuf:"varint,7,opt,name=Remote,proto3" json:"Remote,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } - -func (m *QueryRequest) GetDB() string { - if m != nil && m.DB != nil { - return *m.DB - } - return "" -} - -func (m *QueryRequest) GetQuery() string { - if m != nil && m.Query != nil { - return *m.Query - } - return "" -} - -func (m *QueryRequest) GetSlices() []uint64 { - if m != nil { - return m.Slices - } - return nil -} - -func (m *QueryRequest) GetProfiles() bool { - if m != nil && m.Profiles != nil { - return *m.Profiles - } - return false -} - -func (m *QueryRequest) GetTimestamp() int64 { - if m != nil && m.Timestamp != nil { - return *m.Timestamp - } - return 0 -} - -func (m *QueryRequest) GetQuantum() uint32 { - if m != nil && m.Quantum != nil { - return *m.Quantum - } - return 0 -} - -func (m *QueryRequest) GetRemote() bool { - if m != nil && m.Remote != nil { - return *m.Remote - } - return false -} +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } type QueryResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` - Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` - XXX_unrecognized []byte `json:"-"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` } func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} } - -func (m *QueryResponse) GetErr() string { - if m != nil && m.Err != nil { - return *m.Err - } - return "" -} +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} } func (m *QueryResponse) GetResults() []*QueryResult { if m != nil { @@ -297,17 +188,16 @@ func (m *QueryResponse) GetProfiles() []*Profile { } type QueryResult struct { - Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` - N *uint64 `protobuf:"varint,2,opt,name=N" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed *bool `protobuf:"varint,4,opt,name=Changed" json:"Changed,omitempty"` - XXX_unrecognized []byte `json:"-"` + Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} } func (m *QueryResult) GetBitmap() *Bitmap { if m != nil { @@ -316,13 +206,6 @@ func (m *QueryResult) GetBitmap() *Bitmap { return nil } -func (m *QueryResult) GetN() uint64 { - if m != nil && m.N != nil { - return *m.N - } - return 0 -} - func (m *QueryResult) GetPairs() []*Pair { if m != nil { return m.Pairs @@ -330,180 +213,71 @@ func (m *QueryResult) GetPairs() []*Pair { return nil } -func (m *QueryResult) GetChanged() bool { - if m != nil && m.Changed != nil { - return *m.Changed - } - return false -} - type ImportRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Frame *string `protobuf:"bytes,2,req,name=Frame" json:"Frame,omitempty"` - Slice *uint64 `protobuf:"varint,3,req,name=Slice" json:"Slice,omitempty"` - BitmapIDs []uint64 `protobuf:"varint,4,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,5,rep,name=ProfileIDs" json:"ProfileIDs,omitempty"` - XXX_unrecognized []byte `json:"-"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` } func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} } - -func (m *ImportRequest) GetDB() string { - if m != nil && m.DB != nil { - return *m.DB - } - return "" -} - -func (m *ImportRequest) GetFrame() string { - if m != nil && m.Frame != nil { - return *m.Frame - } - return "" -} - -func (m *ImportRequest) GetSlice() uint64 { - if m != nil && m.Slice != nil { - return *m.Slice - } - return 0 -} - -func (m *ImportRequest) GetBitmapIDs() []uint64 { - if m != nil { - return m.BitmapIDs - } - return nil -} - -func (m *ImportRequest) GetProfileIDs() []uint64 { - if m != nil { - return m.ProfileIDs - } - return nil -} +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } type ImportResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"` - XXX_unrecognized []byte `json:"-"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` } func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} -func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} } - -func (m *ImportResponse) GetErr() string { - if m != nil && m.Err != nil { - return *m.Err - } - return "" -} +func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } type BlockDataRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Frame *string `protobuf:"bytes,2,req,name=Frame" json:"Frame,omitempty"` - Slice *uint64 `protobuf:"varint,3,req,name=Slice" json:"Slice,omitempty"` - Block *uint64 `protobuf:"varint,4,req,name=Block" json:"Block,omitempty"` - XXX_unrecognized []byte `json:"-"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` } func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} -func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } - -func (m *BlockDataRequest) GetDB() string { - if m != nil && m.DB != nil { - return *m.DB - } - return "" -} - -func (m *BlockDataRequest) GetFrame() string { - if m != nil && m.Frame != nil { - return *m.Frame - } - return "" -} - -func (m *BlockDataRequest) GetSlice() uint64 { - if m != nil && m.Slice != nil { - return *m.Slice - } - return 0 -} - -func (m *BlockDataRequest) GetBlock() uint64 { - if m != nil && m.Block != nil { - return *m.Block - } - return 0 -} +func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } type BlockDataResponse struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,2,rep,name=ProfileIDs" json:"ProfileIDs,omitempty"` - XXX_unrecognized []byte `json:"-"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` } func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} -func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } - -func (m *BlockDataResponse) GetBitmapIDs() []uint64 { - if m != nil { - return m.BitmapIDs - } - return nil -} - -func (m *BlockDataResponse) GetProfileIDs() []uint64 { - if m != nil { - return m.ProfileIDs - } - return nil -} +func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } type Cache struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` - XXX_unrecognized []byte `json:"-"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` } func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} -func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } - -func (m *Cache) GetBitmapIDs() []uint64 { - if m != nil { - return m.BitmapIDs - } - return nil -} +func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{15} } type SliceMaxResponse struct { - SliceMax *uint64 `protobuf:"varint,1,req,name=SliceMax" json:"SliceMax,omitempty"` - XXX_unrecognized []byte `json:"-"` + SliceMax uint64 `protobuf:"varint,1,opt,name=SliceMax,proto3" json:"SliceMax,omitempty"` } func (m *SliceMaxResponse) Reset() { *m = SliceMaxResponse{} } func (m *SliceMaxResponse) String() string { return proto.CompactTextString(m) } func (*SliceMaxResponse) ProtoMessage() {} -func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } - -func (m *SliceMaxResponse) GetSliceMax() uint64 { - if m != nil && m.SliceMax != nil { - return *m.SliceMax - } - return 0 -} +func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{16} } func init() { + proto.RegisterType((*DB)(nil), "internal.DB") + proto.RegisterType((*Frame)(nil), "internal.Frame") proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*Bit)(nil), "internal.Bit") @@ -520,39 +294,3384 @@ func init() { proto.RegisterType((*Cache)(nil), "internal.Cache") proto.RegisterType((*SliceMaxResponse)(nil), "internal.SliceMaxResponse") } +func (m *DB) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DB) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } + return i, nil +} + +func (m *Frame) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Frame) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } + return i, nil +} + +func (m *Bitmap) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Bitmap) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Bits) > 0 { + dAtA2 := make([]byte, len(m.Bits)*10) + var j1 int + for _, num := range m.Bits { + for num >= 1<<7 { + dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j1++ + } + dAtA2[j1] = uint8(num) + j1++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(j1)) + i += copy(dAtA[i:], dAtA2[:j1]) + } + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Pair) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Pair) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Key != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Key)) + } + if m.Count != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Count)) + } + return i, nil +} + +func (m *Bit) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Bit) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.BitmapID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.BitmapID)) + } + if m.ProfileID != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.ProfileID)) + } + return i, nil +} + +func (m *Profile) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Profile) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.ID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.ID)) + } + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Attr) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Attr) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Key) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } + if m.Type != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Type)) + } + if len(m.StringValue) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.StringValue))) + i += copy(dAtA[i:], m.StringValue) + } + if m.UintValue != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.UintValue)) + } + if m.BoolValue { + dAtA[i] = 0x28 + i++ + if m.BoolValue { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *AttrMap) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AttrMap) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *QueryRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Query) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Query))) + i += copy(dAtA[i:], m.Query) + } + if len(m.Slices) > 0 { + dAtA4 := make([]byte, len(m.Slices)*10) + var j3 int + for _, num := range m.Slices { + for num >= 1<<7 { + dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j3++ + } + dAtA4[j3] = uint8(num) + j3++ + } + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(j3)) + i += copy(dAtA[i:], dAtA4[:j3]) + } + if m.Profiles { + dAtA[i] = 0x20 + i++ + if m.Profiles { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.Timestamp != 0 { + dAtA[i] = 0x28 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Timestamp)) + } + if len(m.Quantum) > 0 { + dAtA[i] = 0x32 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Quantum))) + i += copy(dAtA[i:], m.Quantum) + } + if m.Remote { + dAtA[i] = 0x38 + i++ + if m.Remote { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *QueryResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Err) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Err))) + i += copy(dAtA[i:], m.Err) + } + if len(m.Results) > 0 { + for _, msg := range m.Results { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.Profiles) > 0 { + for _, msg := range m.Profiles { + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *QueryResult) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Bitmap != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Bitmap.Size())) + n5, err := m.Bitmap.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n5 + } + if m.N != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.N)) + } + if len(m.Pairs) > 0 { + for _, msg := range m.Pairs { + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Changed { + dAtA[i] = 0x20 + i++ + if m.Changed { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *ImportRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Slice != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Slice)) + } + if len(m.BitmapIDs) > 0 { + dAtA7 := make([]byte, len(m.BitmapIDs)*10) + var j6 int + for _, num := range m.BitmapIDs { + for num >= 1<<7 { + dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j6++ + } + dAtA7[j6] = uint8(num) + j6++ + } + dAtA[i] = 0x22 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j6)) + i += copy(dAtA[i:], dAtA7[:j6]) + } + if len(m.ProfileIDs) > 0 { + dAtA9 := make([]byte, len(m.ProfileIDs)*10) + var j8 int + for _, num := range m.ProfileIDs { + for num >= 1<<7 { + dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j8++ + } + dAtA9[j8] = uint8(num) + j8++ + } + dAtA[i] = 0x2a + i++ + i = encodeVarintInternal(dAtA, i, uint64(j8)) + i += copy(dAtA[i:], dAtA9[:j8]) + } + return i, nil +} + +func (m *ImportResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Err) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Err))) + i += copy(dAtA[i:], m.Err) + } + return i, nil +} + +func (m *BlockDataRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Slice != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Slice)) + } + if m.Block != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Block)) + } + return i, nil +} + +func (m *BlockDataResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.BitmapIDs) > 0 { + dAtA11 := make([]byte, len(m.BitmapIDs)*10) + var j10 int + for _, num := range m.BitmapIDs { + for num >= 1<<7 { + dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j10++ + } + dAtA11[j10] = uint8(num) + j10++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(j10)) + i += copy(dAtA[i:], dAtA11[:j10]) + } + if len(m.ProfileIDs) > 0 { + dAtA13 := make([]byte, len(m.ProfileIDs)*10) + var j12 int + for _, num := range m.ProfileIDs { + for num >= 1<<7 { + dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j12++ + } + dAtA13[j12] = uint8(num) + j12++ + } + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j12)) + i += copy(dAtA[i:], dAtA13[:j12]) + } + return i, nil +} + +func (m *Cache) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Cache) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.BitmapIDs) > 0 { + dAtA15 := make([]byte, len(m.BitmapIDs)*10) + var j14 int + for _, num := range m.BitmapIDs { + for num >= 1<<7 { + dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j14++ + } + dAtA15[j14] = uint8(num) + j14++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(j14)) + i += copy(dAtA[i:], dAtA15[:j14]) + } + return i, nil +} + +func (m *SliceMaxResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SliceMaxResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.SliceMax != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.SliceMax)) + } + return i, nil +} + +func encodeFixed64Internal(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Internal(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintInternal(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *DB) Size() (n int) { + var l int + _ = l + l = len(m.TimeQuantum) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + return n +} + +func (m *Frame) Size() (n int) { + var l int + _ = l + l = len(m.TimeQuantum) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + return n +} + +func (m *Bitmap) Size() (n int) { + var l int + _ = l + if len(m.Bits) > 0 { + l = 0 + for _, e := range m.Bits { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if len(m.Attrs) > 0 { + for _, e := range m.Attrs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *Pair) Size() (n int) { + var l int + _ = l + if m.Key != 0 { + n += 1 + sovInternal(uint64(m.Key)) + } + if m.Count != 0 { + n += 1 + sovInternal(uint64(m.Count)) + } + return n +} + +func (m *Bit) Size() (n int) { + var l int + _ = l + if m.BitmapID != 0 { + n += 1 + sovInternal(uint64(m.BitmapID)) + } + if m.ProfileID != 0 { + n += 1 + sovInternal(uint64(m.ProfileID)) + } + return n +} + +func (m *Profile) Size() (n int) { + var l int + _ = l + if m.ID != 0 { + n += 1 + sovInternal(uint64(m.ID)) + } + if len(m.Attrs) > 0 { + for _, e := range m.Attrs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *Attr) Size() (n int) { + var l int + _ = l + l = len(m.Key) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Type != 0 { + n += 1 + sovInternal(uint64(m.Type)) + } + l = len(m.StringValue) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.UintValue != 0 { + n += 1 + sovInternal(uint64(m.UintValue)) + } + if m.BoolValue { + n += 2 + } + return n +} + +func (m *AttrMap) Size() (n int) { + var l int + _ = l + if len(m.Attrs) > 0 { + for _, e := range m.Attrs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *QueryRequest) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + l = len(m.Query) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if len(m.Slices) > 0 { + l = 0 + for _, e := range m.Slices { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if m.Profiles { + n += 2 + } + if m.Timestamp != 0 { + n += 1 + sovInternal(uint64(m.Timestamp)) + } + l = len(m.Quantum) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Remote { + n += 2 + } + return n +} + +func (m *QueryResponse) Size() (n int) { + var l int + _ = l + l = len(m.Err) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + if len(m.Profiles) > 0 { + for _, e := range m.Profiles { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *QueryResult) Size() (n int) { + var l int + _ = l + if m.Bitmap != nil { + l = m.Bitmap.Size() + n += 1 + l + sovInternal(uint64(l)) + } + if m.N != 0 { + n += 1 + sovInternal(uint64(m.N)) + } + if len(m.Pairs) > 0 { + for _, e := range m.Pairs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + if m.Changed { + n += 2 + } + return n +} + +func (m *ImportRequest) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovInternal(uint64(m.Slice)) + } + if len(m.BitmapIDs) > 0 { + l = 0 + for _, e := range m.BitmapIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if len(m.ProfileIDs) > 0 { + l = 0 + for _, e := range m.ProfileIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + return n +} + +func (m *ImportResponse) Size() (n int) { + var l int + _ = l + l = len(m.Err) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + return n +} + +func (m *BlockDataRequest) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovInternal(uint64(m.Slice)) + } + if m.Block != 0 { + n += 1 + sovInternal(uint64(m.Block)) + } + return n +} + +func (m *BlockDataResponse) Size() (n int) { + var l int + _ = l + if len(m.BitmapIDs) > 0 { + l = 0 + for _, e := range m.BitmapIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if len(m.ProfileIDs) > 0 { + l = 0 + for _, e := range m.ProfileIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + return n +} + +func (m *Cache) Size() (n int) { + var l int + _ = l + if len(m.BitmapIDs) > 0 { + l = 0 + for _, e := range m.BitmapIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + return n +} + +func (m *SliceMaxResponse) Size() (n int) { + var l int + _ = l + if m.SliceMax != 0 { + n += 1 + sovInternal(uint64(m.SliceMax)) + } + return n +} + +func sovInternal(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozInternal(x uint64) (n int) { + return sovInternal(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *DB) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DB: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DB: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Frame) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Frame: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Bitmap) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Bitmap: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Bitmap: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Bits = append(m.Bits, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Bits = append(m.Bits, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType) + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attrs = append(m.Attrs, &Attr{}) + if err := m.Attrs[len(m.Attrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Pair) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Pair: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Pair: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + m.Key = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Key |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) + } + m.Count = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Count |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Bit) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Bit: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Bit: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapID", wireType) + } + m.BitmapID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BitmapID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProfileID", wireType) + } + m.ProfileID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProfileID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Profile) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Profile: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Profile: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + m.ID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attrs = append(m.Attrs, &Attr{}) + if err := m.Attrs[len(m.Attrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Attr) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Attr: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Attr: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + m.Type = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Type |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StringValue", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.StringValue = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UintValue", wireType) + } + m.UintValue = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UintValue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BoolValue", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BoolValue = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *AttrMap) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: AttrMap: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: AttrMap: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attrs = append(m.Attrs, &Attr{}) + if err := m.Attrs[len(m.Attrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Query", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Query = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Profiles = bool(v != 0) + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) + } + m.Timestamp = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Timestamp |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Quantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Quantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Remote", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Remote = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Err", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Err = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &QueryResult{}) + if err := m.Results[len(m.Results)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Profiles = append(m.Profiles, &Profile{}) + if err := m.Profiles[len(m.Profiles)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryResult) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Bitmap", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Bitmap == nil { + m.Bitmap = &Bitmap{} + } + if err := m.Bitmap.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field N", wireType) + } + m.N = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.N |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pairs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Pairs = append(m.Pairs, &Pair{}) + if err := m.Pairs[len(m.Pairs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Changed", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Changed = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ImportRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + } + case 5: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ImportResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Err", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Err = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockDataRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockDataRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Block", wireType) + } + m.Block = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Block |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockDataResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockDataResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + } + case 2: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Cache) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Cache: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Cache: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SliceMaxResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SliceMaxResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SliceMaxResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SliceMax", wireType) + } + m.SliceMax = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SliceMax |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipInternal(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthInternal + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipInternal(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthInternal = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowInternal = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 499 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x6b, 0xd4, 0x40, - 0x14, 0x65, 0x37, 0xd9, 0xcd, 0xee, 0x4d, 0x77, 0xdd, 0x1d, 0x11, 0x83, 0x50, 0x28, 0x51, 0xa4, - 0xf8, 0x50, 0xa1, 0xe8, 0x8b, 0x6f, 0x6e, 0x57, 0xa1, 0x94, 0x96, 0x7e, 0xa8, 0xcf, 0x0e, 0xeb, - 0xd8, 0x46, 0x93, 0x4c, 0x9c, 0x4c, 0xc0, 0x3e, 0xf9, 0xd7, 0x3d, 0x33, 0x99, 0xc9, 0x46, 0x58, - 0x11, 0x9f, 0x92, 0x7b, 0xe6, 0x9e, 0x7b, 0xce, 0x9c, 0xdc, 0xd0, 0xe3, 0xac, 0xd4, 0x42, 0x95, - 0x3c, 0x7f, 0xe9, 0x5f, 0x8e, 0x2a, 0x25, 0xb5, 0x64, 0x13, 0x5f, 0xa7, 0xaf, 0x69, 0xbc, 0xca, - 0x74, 0xc1, 0x2b, 0xb6, 0x47, 0x21, 0xde, 0xea, 0x64, 0x70, 0x10, 0x1c, 0x86, 0x6c, 0x9f, 0x46, - 0x6f, 0xb5, 0x56, 0x75, 0x32, 0x44, 0x19, 0x1f, 0xcf, 0x8f, 0xba, 0x09, 0x06, 0x4e, 0x53, 0x0a, - 0x2f, 0x79, 0xa6, 0x58, 0x4c, 0xc1, 0x99, 0xb8, 0x07, 0x67, 0x08, 0xce, 0x8c, 0x46, 0x27, 0xb2, - 0x29, 0x35, 0x38, 0x28, 0xd3, 0x17, 0x14, 0x60, 0x20, 0x5b, 0xd0, 0xa4, 0x55, 0x38, 0x5d, 0xbb, - 0xbe, 0x25, 0x4d, 0x2f, 0x95, 0xfc, 0x9a, 0xe5, 0x02, 0x50, 0xdb, 0xfb, 0x8a, 0x22, 0x07, 0x31, - 0xa2, 0x61, 0xd7, 0xf9, 0x0f, 0x17, 0x17, 0x14, 0x9a, 0x67, 0xdf, 0xc5, 0x94, 0x3d, 0xa4, 0xf8, - 0x46, 0xab, 0xac, 0xbc, 0xfd, 0xc4, 0xf3, 0x46, 0x80, 0x39, 0x00, 0x08, 0xc9, 0x8f, 0xe0, 0xb6, - 0x50, 0x00, 0xc8, 0xba, 0x58, 0x49, 0x99, 0xb7, 0x50, 0x08, 0x68, 0x92, 0x1e, 0x52, 0x64, 0xe6, - 0x9d, 0x23, 0x8d, 0x4e, 0x79, 0xb0, 0x53, 0xf9, 0x17, 0xed, 0x5d, 0x35, 0x42, 0xdd, 0x5f, 0x8b, - 0x1f, 0x8d, 0xa8, 0xb5, 0x31, 0xbd, 0x5e, 0x39, 0x03, 0x88, 0xc1, 0x9e, 0xd9, 0xab, 0x4d, 0xd9, - 0x9c, 0xc6, 0x37, 0x79, 0xb6, 0x11, 0x35, 0x74, 0x4d, 0xb2, 0xc8, 0xc3, 0x5d, 0xb5, 0x6e, 0x65, - 0x8d, 0x93, 0x0f, 0x59, 0x81, 0x31, 0xbc, 0xa8, 0x92, 0x11, 0xa0, 0x80, 0x3d, 0xa0, 0xe8, 0xaa, - 0xe1, 0xa5, 0x6e, 0x8a, 0x64, 0x0c, 0x60, 0x66, 0xa6, 0x5c, 0x8b, 0x42, 0x6a, 0x91, 0x44, 0xd6, - 0x6a, 0x46, 0x33, 0x67, 0xa0, 0xae, 0x64, 0x59, 0x0b, 0x93, 0xc1, 0x3b, 0xa5, 0x60, 0xc1, 0x5c, - 0xf7, 0x39, 0x45, 0x38, 0x68, 0x72, 0xed, 0x93, 0x7b, 0xb4, 0xf5, 0xef, 0x69, 0x38, 0x65, 0x4f, - 0x7b, 0x5e, 0x02, 0xdb, 0xb8, 0xdc, 0x36, 0xba, 0x93, 0xf4, 0x1b, 0xc5, 0x7d, 0xce, 0x81, 0xdf, - 0x18, 0xab, 0x15, 0x1f, 0x2f, 0xb6, 0x0c, 0xb7, 0x49, 0x53, 0x1a, 0x5c, 0xd8, 0xdc, 0xed, 0x07, - 0x34, 0x7b, 0xe2, 0xa7, 0xf7, 0x62, 0xb4, 0xeb, 0x83, 0x6b, 0x9e, 0xdc, 0xf1, 0xf2, 0x56, 0x7c, - 0x71, 0x5f, 0xe0, 0x33, 0xcd, 0x4e, 0x8b, 0x4a, 0x2a, 0xfd, 0x97, 0x60, 0xdf, 0x2b, 0x5e, 0x08, - 0x17, 0x2c, 0x4a, 0x1b, 0x2c, 0x66, 0xbb, 0xad, 0xf2, 0x7b, 0x66, 0x82, 0x35, 0x51, 0x83, 0xdd, - 0x2d, 0x5a, 0x8d, 0x64, 0x81, 0xa5, 0xfb, 0x34, 0xf7, 0x0a, 0x3b, 0x92, 0x4b, 0xcf, 0x68, 0xb1, - 0xca, 0xe5, 0xe6, 0xfb, 0x9a, 0x6b, 0xfe, 0xff, 0x1e, 0x50, 0x5a, 0x36, 0xf4, 0xcd, 0x56, 0xbf, - 0xa1, 0x65, 0x6f, 0x98, 0x93, 0xfb, 0xc3, 0xe7, 0x60, 0x87, 0xcf, 0xa1, 0xf5, 0xf9, 0x04, 0x3f, - 0x13, 0xdf, 0xdc, 0xed, 0xea, 0x4f, 0x9f, 0xd1, 0xc2, 0xaa, 0x9e, 0xf3, 0x9f, 0xdd, 0x58, 0xac, - 0x95, 0xc7, 0xda, 0x9f, 0xe7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x12, 0x50, 0x70, 0xfe, - 0x03, 0x00, 0x00, + // 638 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x54, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0x66, 0x63, 0x3b, 0x3f, 0x93, 0x36, 0x4a, 0x57, 0x01, 0x59, 0x15, 0x8a, 0xa2, 0x15, 0xa0, + 0x70, 0x20, 0x95, 0xca, 0x03, 0x54, 0xb8, 0x29, 0x52, 0x84, 0x5a, 0xb5, 0xdb, 0xc2, 0x7d, 0x69, + 0x97, 0xd6, 0xc2, 0x7f, 0xac, 0xd7, 0x12, 0x3d, 0x72, 0xe0, 0x00, 0x4f, 0xc0, 0xa3, 0xf0, 0x08, + 0x1c, 0x79, 0x04, 0x54, 0x5e, 0x04, 0xed, 0x9f, 0x6d, 0x84, 0x08, 0x3d, 0x70, 0xdb, 0xef, 0x1b, + 0xef, 0xce, 0x37, 0xf3, 0xcd, 0x18, 0x46, 0x71, 0x26, 0xb9, 0xc8, 0x58, 0xb2, 0x28, 0x44, 0x2e, + 0x73, 0xdc, 0x77, 0x98, 0x3c, 0x82, 0xce, 0x32, 0xc2, 0x33, 0x18, 0x9e, 0xc5, 0x29, 0x3f, 0xa9, + 0x58, 0x26, 0xab, 0x34, 0x44, 0x33, 0x34, 0x1f, 0xd0, 0x36, 0x45, 0x1e, 0x43, 0xf0, 0x5c, 0xb0, + 0x94, 0xdf, 0xe2, 0xd3, 0x08, 0xba, 0x51, 0x2c, 0x53, 0x56, 0x60, 0x0c, 0x7e, 0x14, 0xcb, 0x32, + 0x44, 0x33, 0x6f, 0xee, 0x53, 0x7d, 0xc6, 0x0f, 0x20, 0x78, 0x26, 0xa5, 0x28, 0xc3, 0xce, 0xcc, + 0x9b, 0x0f, 0x77, 0x47, 0x8b, 0x5a, 0x9a, 0xa2, 0xa9, 0x09, 0x92, 0x05, 0xf8, 0xc7, 0x2c, 0x16, + 0x78, 0x0c, 0xde, 0x0b, 0x7e, 0xad, 0xb3, 0xf8, 0x54, 0x1d, 0xf1, 0x04, 0x82, 0xfd, 0xbc, 0xca, + 0x64, 0xd8, 0xd1, 0x9c, 0x01, 0x64, 0x0f, 0xbc, 0x28, 0x96, 0x78, 0x1b, 0xfa, 0x26, 0xf5, 0x6a, + 0x69, 0xef, 0xd4, 0x18, 0xdf, 0x87, 0xc1, 0xb1, 0xc8, 0xdf, 0xc4, 0x09, 0x5f, 0x2d, 0xed, 0xe5, + 0x86, 0x20, 0x7b, 0xd0, 0xb3, 0x00, 0x8f, 0xa0, 0x53, 0x5f, 0xef, 0xac, 0x96, 0xb7, 0x54, 0xfc, + 0x19, 0x81, 0xaf, 0x4e, 0x6d, 0xc9, 0x03, 0x23, 0x19, 0x83, 0x7f, 0x76, 0x5d, 0x70, 0x9b, 0x54, + 0x9f, 0x55, 0x1b, 0x4f, 0xa5, 0x88, 0xb3, 0xcb, 0x57, 0x2c, 0xa9, 0x78, 0xe8, 0x99, 0x36, 0xb6, + 0x28, 0xa5, 0xf7, 0x65, 0x9c, 0x49, 0x13, 0xf7, 0x8d, 0xde, 0x9a, 0x50, 0xd1, 0x28, 0xcf, 0x13, + 0x13, 0x0d, 0x66, 0x68, 0xde, 0xa7, 0x0d, 0x41, 0x76, 0xa0, 0xa7, 0xb4, 0x1c, 0xb2, 0xa2, 0x51, + 0x8f, 0xd6, 0xa9, 0xff, 0x8a, 0x60, 0xe3, 0xa4, 0xe2, 0xe2, 0x9a, 0xf2, 0x77, 0x15, 0x2f, 0xa5, + 0x6a, 0xc2, 0x32, 0xb2, 0x45, 0xa8, 0x09, 0x99, 0x40, 0xa0, 0xe3, 0xba, 0x88, 0x01, 0x35, 0x00, + 0xdf, 0x83, 0xee, 0x69, 0x12, 0x9f, 0xf3, 0x32, 0xf4, 0xb4, 0xc5, 0x16, 0x29, 0x1f, 0x6c, 0x37, + 0x4b, 0x2d, 0xbd, 0x4f, 0x6b, 0xac, 0x94, 0xab, 0x69, 0x29, 0x25, 0x4b, 0x0b, 0xad, 0xdc, 0xa3, + 0x0d, 0x81, 0x43, 0xe8, 0xb9, 0xd1, 0xea, 0xea, 0x4c, 0x0e, 0xaa, 0x5c, 0x94, 0xa7, 0xb9, 0xe4, + 0x61, 0x4f, 0xbf, 0x68, 0x11, 0xf9, 0x80, 0x60, 0xd3, 0x4a, 0x2f, 0x8b, 0x3c, 0x2b, 0xb9, 0x72, + 0xe0, 0x40, 0x08, 0xe7, 0xc0, 0x81, 0x10, 0x78, 0x07, 0x7a, 0x94, 0x97, 0x55, 0x22, 0x9d, 0x89, + 0x77, 0x9b, 0x36, 0xb8, 0xbb, 0x55, 0x22, 0xa9, 0xfb, 0x0a, 0x3f, 0x69, 0x15, 0xe0, 0xe9, 0x1b, + 0x5b, 0xcd, 0x0d, 0x1b, 0x69, 0x6a, 0x22, 0x1f, 0x11, 0x0c, 0x5b, 0xef, 0xe0, 0xb9, 0x5b, 0x01, + 0x2d, 0x62, 0xb8, 0x3b, 0x6e, 0x2e, 0x1b, 0x9e, 0xba, 0x15, 0xd9, 0x00, 0x74, 0x64, 0x07, 0x03, + 0x1d, 0x29, 0xb3, 0xd4, 0xd8, 0xbb, 0x9c, 0x2d, 0xb3, 0x14, 0x4d, 0x4d, 0x50, 0xf5, 0x68, 0xff, + 0x8a, 0x65, 0x97, 0xfc, 0xc2, 0x36, 0xd7, 0x41, 0xf2, 0x09, 0xc1, 0xe6, 0x2a, 0x2d, 0x72, 0x21, + 0xd7, 0xf8, 0xa8, 0xf7, 0xd8, 0xf9, 0x68, 0x96, 0x7a, 0x02, 0x81, 0x76, 0x4e, 0xcf, 0xa1, 0x4f, + 0x0d, 0xd0, 0x33, 0x66, 0xb7, 0x47, 0xd9, 0xa8, 0x0c, 0x6e, 0x08, 0x3c, 0x05, 0xa8, 0xd7, 0xa7, + 0x0c, 0x03, 0x1d, 0x6e, 0x31, 0x84, 0xc0, 0xc8, 0x49, 0xf9, 0x9b, 0x2f, 0xe4, 0x02, 0xc6, 0x51, + 0x92, 0x9f, 0xbf, 0x5d, 0x32, 0xc9, 0xfe, 0x87, 0xe2, 0x09, 0x04, 0xfa, 0x3d, 0xbb, 0x2f, 0x06, + 0x90, 0x13, 0xd8, 0x6a, 0x65, 0xb1, 0x62, 0x7e, 0x2b, 0x0e, 0xad, 0x2f, 0xae, 0xf3, 0x47, 0x71, + 0x0f, 0x21, 0xd8, 0x67, 0xe7, 0x57, 0xff, 0x78, 0x86, 0x2c, 0x60, 0xac, 0x85, 0x1d, 0xb2, 0xf7, + 0x75, 0xe2, 0x6d, 0xe8, 0x3b, 0xce, 0xfd, 0xa3, 0x1c, 0x8e, 0xc6, 0xdf, 0x6e, 0xa6, 0xe8, 0xfb, + 0xcd, 0x14, 0xfd, 0xb8, 0x99, 0xa2, 0x2f, 0x3f, 0xa7, 0x77, 0x5e, 0x77, 0xf5, 0x0f, 0xfb, 0xe9, + 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x54, 0x5a, 0xc6, 0x13, 0xc2, 0x05, 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index 79e1d627d..3c426e6eb 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -1,30 +1,41 @@ +syntax = "proto3"; + package internal; +message DB { + string TimeQuantum = 1; +} + +message Frame { + string TimeQuantum = 1; +} + message Bitmap { - repeated uint64 Bits = 1; - repeated Attr Attrs = 2; + repeated uint64 Bits = 1; + repeated Attr Attrs = 2; } message Pair { - required uint64 Key = 1; - required uint64 Count = 2; + uint64 Key = 1; + uint64 Count = 2; } message Bit { - required uint64 BitmapID = 1; - required uint64 ProfileID = 2; + uint64 BitmapID = 1; + uint64 ProfileID = 2; } message Profile { - required uint64 ID = 1; + uint64 ID = 1; repeated Attr Attrs = 2; } message Attr { - required string Key = 1; - optional string StringValue = 2; - optional uint64 UintValue = 3; - optional bool BoolValue = 4; + string Key = 1; + uint64 Type = 2; + string StringValue = 3; + uint64 UintValue = 4; + bool BoolValue = 5; } message AttrMap { @@ -32,49 +43,49 @@ message AttrMap { } message QueryRequest { - required string DB = 1; - required string Query = 2; - repeated uint64 Slices = 3; - optional bool Profiles = 4; - optional int64 Timestamp = 5; - optional uint32 Quantum = 6; - optional bool Remote = 7; + string DB = 1; + string Query = 2; + repeated uint64 Slices = 3; + bool Profiles = 4; + int64 Timestamp = 5; + string Quantum = 6; + bool Remote = 7; } message QueryResponse { - optional string Err = 1; - repeated QueryResult Results = 2; - repeated Profile Profiles = 3; + string Err = 1; + repeated QueryResult Results = 2; + repeated Profile Profiles = 3; } message QueryResult { - optional Bitmap Bitmap = 1; - optional uint64 N = 2; - repeated Pair Pairs = 3; - optional bool Changed = 4; + Bitmap Bitmap = 1; + uint64 N = 2; + repeated Pair Pairs = 3; + bool Changed = 4; } message ImportRequest { - required string DB = 1; - required string Frame = 2; - required uint64 Slice = 3; - repeated uint64 BitmapIDs = 4; + string DB = 1; + string Frame = 2; + uint64 Slice = 3; + repeated uint64 BitmapIDs = 4; repeated uint64 ProfileIDs = 5; } message ImportResponse { - optional string Err = 1; + string Err = 1; } message BlockDataRequest { - required string DB = 1; - required string Frame = 2; - required uint64 Slice = 3; - required uint64 Block = 4; + string DB = 1; + string Frame = 2; + uint64 Slice = 3; + uint64 Block = 4; } message BlockDataResponse { - repeated uint64 BitmapIDs = 1; + repeated uint64 BitmapIDs = 1; repeated uint64 ProfileIDs = 2; } @@ -83,5 +94,6 @@ message Cache { } message SliceMaxResponse { - required uint64 SliceMax = 1; + uint64 SliceMax = 1; } + diff --git a/pilosa.go b/pilosa.go index 4d1fbba41..d6c07bb0c 100644 --- a/pilosa.go +++ b/pilosa.go @@ -3,12 +3,9 @@ package pilosa import ( "errors" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) -//go:generate protoc --gogo_out=. internal/internal.proto - var ( // ErrHostRequired is returned when excuting a remote operation without a host. ErrHostRequired = errors.New("host required") @@ -57,7 +54,7 @@ func decodeProfiles(a []*internal.Profile) []*Profile { // encodeProfile converts p into its internal representation. func encodeProfile(p *Profile) *internal.Profile { return &internal.Profile{ - ID: proto.Uint64(p.ID), + ID: p.ID, Attrs: encodeAttrs(p.Attrs), } } @@ -65,12 +62,12 @@ func encodeProfile(p *Profile) *internal.Profile { // decodeProfile converts b from its internal representation. func decodeProfile(pb *internal.Profile) *Profile { p := &Profile{ - ID: pb.GetID(), + ID: pb.ID, } - if len(pb.GetAttrs()) > 0 { - p.Attrs = make(map[string]interface{}, len(pb.GetAttrs())) - for _, attr := range pb.GetAttrs() { + if len(pb.Attrs) > 0 { + p.Attrs = make(map[string]interface{}, len(pb.Attrs)) + for _, attr := range pb.Attrs { k, v := decodeAttr(attr) p.Attrs[k] = v } diff --git a/server.go b/server.go index 635ed0e10..670ed707d 100644 --- a/server.go +++ b/server.go @@ -255,6 +255,5 @@ func checkMaxSlice(hostport string) (uint64, error) { return 0, err } - return *pb.SliceMax, nil - + return pb.SliceMax, nil } diff --git a/time.go b/time.go index ce11ea1d9..b117023ad 100644 --- a/time.go +++ b/time.go @@ -2,122 +2,166 @@ package pilosa import ( "errors" + "fmt" "strings" "time" ) -// TimeQuantum represents a time granularity for time-based bitmap ids. -type TimeQuantum uint +// ErrInvalidTimeQuantum is returned when parsing a time quantum. +var ErrInvalidTimeQuantum = errors.New("invalid time quantum") -// ParseTimeQuantum parses s into a quantum. -func ParseTimeQuantum(s string) (TimeQuantum, error) { - switch strings.ToUpper(s) { - case "Y": - return Y, nil - case "M": - return YM, nil - case "D": - return YMD, nil - case "H": - return YMDH, nil - default: - return 0, errors.New("invalid quantum") - } -} +// TimeQuantum represents a time granularity for time-based bitmaps. +type TimeQuantum string -// String returns the string representation of the quantum. -func (q TimeQuantum) String() string { +// HasYear returns true if the quantum contains a 'Y' unit. +func (q TimeQuantum) HasYear() bool { return strings.ContainsRune(string(q), 'Y') } + +// HasMonth returns true if the quantum contains a 'M' unit. +func (q TimeQuantum) HasMonth() bool { return strings.ContainsRune(string(q), 'M') } + +// HasDay returns true if the quantum contains a 'D' unit. +func (q TimeQuantum) HasDay() bool { return strings.ContainsRune(string(q), 'D') } + +// HasHour returns true if the quantum contains a 'H' unit. +func (q TimeQuantum) HasHour() bool { return strings.ContainsRune(string(q), 'H') } + +// Valid returns true if q is a valid time quantum value. +func (q TimeQuantum) Valid() bool { switch q { - case Y: - return "Y" - case YM: - return "M" - case YMD: - return "D" - case YMDH: - return "H" + case "Y", "YM", "YMD", "YMDH", + "M", "MD", "MDH", + "D", "DH", + "H", + "": + return true default: - return "Y" + return false } } -const ( - Y TimeQuantum = 3 - YM TimeQuantum = 2 - YMD TimeQuantum = 1 - YMDH TimeQuantum = 0 -) - -// TimeID returns a packed time identifier from a year/month/day/hour & tile. -func TimeID(q TimeQuantum, year uint, month uint, day uint, hour uint, tileID uint64) uint64 { - v := uint64((uint(q) << 30) | ((year - 1970) << 23) | (month << 19) | (day << 14) | (hour << 9)) - return (v << 32) | tileID +// ParseTimeQuantum parses v into a time quantum. +func ParseTimeQuantum(v string) (TimeQuantum, error) { + q := TimeQuantum(strings.ToUpper(v)) + if !q.Valid() { + return "", ErrInvalidTimeQuantum + } + return q, nil } -// TimeIDsFromRange returns timestamp bitmap ids within a time range. -func TimeIDsFromRange(start, end time.Time, tileID uint64) []uint64 { +// FrameByTimeUnit returns the frame name for time with a given quantum unit. +func FrameByTimeUnit(name string, t time.Time, unit rune) string { + switch unit { + case 'Y': + return fmt.Sprintf("%s_%s", name, t.Format("2006")) + case 'M': + return fmt.Sprintf("%s_%s", name, t.Format("200601")) + case 'D': + return fmt.Sprintf("%s_%s", name, t.Format("20060102")) + case 'H': + return fmt.Sprintf("%s_%s", name, t.Format("2006010215")) + default: + return "" + } +} + +// FramesByTime returns a list of frames for a given timestamp. +func FramesByTime(name string, t time.Time, q TimeQuantum) []string { + a := make([]string, 0, len(q)) + for _, unit := range q { + frame := FrameByTimeUnit(name, t, unit) + if frame == "" { + continue + } + a = append(a, frame) + } + return a +} + +// FramesByTimeRange returns a list of frames to traverse to query a time range. +func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start - var results []uint64 - for t.Before(end) { - if !nextDay(t, end) { + // Save flags for performance. + hasYear := q.HasYear() + hasMonth := q.HasMonth() + hasDay := q.HasDay() + hasHour := q.HasHour() + + var results []string + + // Walk up from smallest units to largest units. + if hasHour || hasDay || hasMonth { + for t.Before(end) { + if hasHour { + if !nextDayGTE(t, end) { + break + } else if t.Hour() != 0 { + results = append(results, FrameByTimeUnit(name, t, 'H')) + t = t.Add(time.Hour) + continue + } + + } + + if hasDay { + if !nextMonthGTE(t, end) { + break + } else if t.Day() != 1 { + results = append(results, FrameByTimeUnit(name, t, 'D')) + t = t.AddDate(0, 0, 1) + continue + } + } + + if hasMonth { + if !nextYearGTE(t, end) { + break + } else if t.Month() != 1 { + results = append(results, FrameByTimeUnit(name, t, 'M')) + t = t.AddDate(0, 1, 0) + continue + } + } + + // If a unit exists but isn't set and there are no larger units + // available then we need to exit the loop because we are no longer + // making progress. break } - if t.Hour() == 0 { - if !nextMonth(t, end) { - break - } - - if t.Day() == 1 { - if !nextYear(t, end) { - break - } - - if t.Month() == 1 { - break - } - - results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) - t = t.AddDate(0, 1, 0) - } else { - results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) - t = t.AddDate(0, 0, 1) - } - } else { - results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) - t = t.Add(time.Hour) - } } + // Walk back down from largest units to smallest units. for t.Before(end) { - if nextYear(t, end) { - results = append(results, TimeID(Y, uint(t.Year()), 0, 0, 0, tileID)) + if hasYear && nextYearGTE(t, end) { + results = append(results, FrameByTimeUnit(name, t, 'Y')) t = t.AddDate(1, 0, 0) - } else if nextMonth(t, end) { - results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) + } else if hasMonth && nextMonthGTE(t, end) { + results = append(results, FrameByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) - } else if nextDay(t, end) { - results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) + } else if hasDay && nextDayGTE(t, end) { + results = append(results, FrameByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) - } else { - results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) + } else if hasHour { + results = append(results, FrameByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) + } else { + break } } return results } -func nextYear(start time.Time, end time.Time) bool { - next := start.AddDate(1, 0, 0) +func nextYearGTE(t time.Time, end time.Time) bool { + next := t.AddDate(1, 0, 0) if next.Year() == end.Year() { return true } return end.After(next) } -func nextMonth(start time.Time, end time.Time) bool { - next := start.AddDate(0, 1, 0) +func nextMonthGTE(t time.Time, end time.Time) bool { + next := t.AddDate(0, 1, 0) y1, m1, _ := next.Date() y2, m2, _ := end.Date() if (y1 == y2) && (m1 == m2) { @@ -126,8 +170,8 @@ func nextMonth(start time.Time, end time.Time) bool { return end.After(next) } -func nextDay(start time.Time, end time.Time) bool { - next := start.AddDate(0, 0, 1) +func nextDayGTE(t time.Time, end time.Time) bool { + next := t.AddDate(0, 0, 1) y1, m1, d1 := next.Date() y2, m2, d2 := end.Date() if (y1 == y2) && (m1 == m2) && (d1 == d2) { @@ -135,23 +179,3 @@ func nextDay(start time.Time, end time.Time) bool { } return end.After(next) } - -// TimeIDsFromQuantum returns a list of time identifiers for a single quantum. -func TimeIDsFromQuantum(q TimeQuantum, t time.Time, tileID uint64) []uint64 { - y, m, d, h := uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()) - - v := make([]uint64, 0, 4) - if q <= Y { - v = append(v, TimeID(Y, y, 0, 0, 0, tileID)) - } - if q <= YM { - v = append(v, TimeID(YM, y, m, 0, 0, tileID)) - } - if q <= YMD { - v = append(v, TimeID(YMD, y, m, d, 0, tileID)) - } - if q <= YMDH { - v = append(v, TimeID(YMDH, y, m, d, h, tileID)) - } - return v -} diff --git a/time_test.go b/time_test.go index aed62849c..28d86555d 100644 --- a/time_test.go +++ b/time_test.go @@ -1,134 +1,162 @@ package pilosa_test import ( + "reflect" "testing" "time" "github.com/pilosa/pilosa" ) -func TestTimeIDsFromRange_1h_0(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-08-11 14:00"), - *MustParseTime("2014-08-11 16:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } +// Ensure string can be parsed into time quantum. +func TestParseTimeQuantum(t *testing.T) { + t.Run("OK", func(t *testing.T) { + if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil { + t.Fatalf("unexpected error: %s", err) + } else if q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %#v", q) + } + }) + + t.Run("ErrInvalidTimeQuantum", func(t *testing.T) { + if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum { + t.Fatalf("unexpected error: %s", err) + } + }) } -func TestTimeIDsFromRange_1h_1(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 10:03"), - *MustParseTime("2014-01-02 11:03"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } +// Ensure generated frame name can be returned for a given time unit. +func TestFrameByTimeUnit(t *testing.T) { + ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) + + t.Run("Y", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'Y'); s != "F_2000" { + t.Fatalf("unexpected name: %s", s) + } + }) + t.Run("M", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'M'); s != "F_200001" { + t.Fatalf("unexpected name: %s", s) + } + }) + t.Run("D", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'D'); s != "F_20000102" { + t.Fatalf("unexpected name: %s", s) + } + }) + t.Run("H", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + t.Fatalf("unexpected name: %s", s) + } + }) } -func TestTimeIDsFromRange_2h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 10:03"), - *MustParseTime("2014-01-02 12:03"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } +// Ensure all applicable frame names can be generated when mutating a time bit. +func TestFramesByTime(t *testing.T) { + ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) + + t.Run("YMDH", func(t *testing.T) { + a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("YMDH")) + if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) { + t.Fatalf("unexpected names: %+v", a) + } + }) + + t.Run("D", func(t *testing.T) { + a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("D")) + if !reflect.DeepEqual(a, []string{"F_20000102"}) { + t.Fatalf("unexpected names: %+v", a) + } + }) } -func TestTimeIDsFromRange_24h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 12:03"), - *MustParseTime("2014-01-03 12:03"), - uint64(1), - ); len(m) != 24 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1d(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 00:00"), - *MustParseTime("2014-01-03 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1d1h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 00:00"), - *MustParseTime("2014-01-03 01:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 23:00"), - *MustParseTime("2014-01-04 00:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d1h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 23:00"), - *MustParseTime("2014-01-04 01:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1y(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-01 00:00"), - *MustParseTime("2015-01-01 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d1m(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-30 23:00"), - *MustParseTime("2014-03-01 00:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d1m1d1h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-30 23:00"), - *MustParseTime("2014-03-02 01:00"), - uint64(1), - ); len(m) != 5 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromQuantum(t *testing.T) { - _ = pilosa.TimeIDsFromQuantum(pilosa.YMD, *MustParseTime("1970-01-01 00:00"), uint64(15027)) +// Ensure sets of frames can be returned for a given time range. +func TestFramesByTimeRange(t *testing.T) { + t.Run("Y", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) + if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("YM", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) + if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("YMD", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) + if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("YMDH", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) + if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("M", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) + if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("MD", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) + if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("MDH", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) + if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("D", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) + if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("DH", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) + if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("H", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) + if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) } // DefaultTimeLayout is the time layout used by the tests. const DefaultTimeLayout = "2006-01-02 15:04" // MustParseTime parses value using DefaultTimeLayout. Panic on error. -func MustParseTime(value string) *time.Time { +func MustParseTime(value string) time.Time { v, err := time.Parse(DefaultTimeLayout, value) if err != nil { panic(err) } + return v +} + +// MustParseTimePtr parses value using DefaultTimeLayout. Panic on error. +func MustParseTimePtr(value string) *time.Time { + v := MustParseTime(value) return &v } + +// MustParseTimeQuantum parses v into a time quantum. Panic on error. +func MustParseTimeQuantum(v string) pilosa.TimeQuantum { + q, err := pilosa.ParseTimeQuantum(v) + if err != nil { + panic(err) + } + return q +}