From b6f0a62fdd7dac71d0fe86a878d1726d6245d44b Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 21 Nov 2016 16:05:28 -0600 Subject: [PATCH 01/11] changes MaxSlices logic to be DB-specific --- client.go | 18 ++-- cluster.go | 2 +- cmd/pilosactl/main.go | 4 +- db.go | 23 +++-- executor.go | 3 +- executor_test.go | 4 +- handler.go | 21 +++-- handler_test.go | 24 +++++ index.go | 36 +++----- index_test.go | 9 +- internal/internal.pb.go | 188 +++++++++++++++++++++------------------- internal/internal.proto | 5 +- server.go | 48 +++++----- 13 files changed, 218 insertions(+), 167 deletions(-) diff --git a/client.go b/client.go index 93efa93d6..9744ae96d 100644 --- a/client.go +++ b/client.go @@ -45,8 +45,8 @@ func NewClient(host string) (*Client, error) { // Host returns the host the client was initialized with. func (c *Client) Host() string { return c.host } -// SliceN returns the number of slices on a server. -func (c *Client) SliceN(ctx context.Context) (uint64, error) { +// SliceNs returns the number of slices on a server by database. +func (c *Client) SliceNs(ctx context.Context) (MaxSlices, error) { // Execute request against the host. u := url.URL{ Scheme: "http", @@ -57,24 +57,24 @@ func (c *Client) SliceN(ctx context.Context) (uint64, error) { // Build request. req, err := http.NewRequest("GET", u.String(), nil) if err != nil { - return 0, err + return nil, err } // Execute request. resp, err := c.HTTPClient.Do(req.WithContext(ctx)) if err != nil { - return 0, err + return nil, err } defer resp.Body.Close() var rsp sliceMaxResponse if resp.StatusCode != http.StatusOK { - return 0, fmt.Errorf("http: status=%d", resp.StatusCode) + return nil, fmt.Errorf("http: status=%d", resp.StatusCode) } else if err := json.NewDecoder(resp.Body).Decode(&rsp); err != nil { - return 0, fmt.Errorf("json decode: %s", err) + return nil, fmt.Errorf("json decode: %s", err) } - return rsp.SliceMax, nil + return rsp.MaxSlices, nil } // Schema returns all database and frame schema information. @@ -362,13 +362,13 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) er tw := tar.NewWriter(w) // Find the maximum number of slices. - sliceN, err := c.SliceN(ctx) + sliceNs, err := c.SliceNs(ctx) if err != nil { return fmt.Errorf("slice n: %s", err) } // Backup every slice to the tar file. - for i := uint64(0); i <= sliceN; i++ { + for i := uint64(0); i <= sliceNs[db]; i++ { if err := c.backupSliceTo(ctx, tw, db, frame, i); err != nil { return err } diff --git a/cluster.go b/cluster.go index 5929f6308..beb33496b 100644 --- a/cluster.go +++ b/cluster.go @@ -31,7 +31,7 @@ func (a Nodes) Contains(n *Node) bool { return false } -// ContainsHost returns true if host matches on of the node's host. +// ContainsHost returns true if host matches one of the node's host. func (a Nodes) ContainsHost(host string) bool { for _, n := range a { if n.Host == host { diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index c51667ee8..fa7986e1e 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -480,13 +480,13 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } // Determine slice count. - sliceN, err := client.SliceN(ctx) + sliceNs, err := client.SliceNs(ctx) if err != nil { return err } // Export each slice. - for slice := uint64(0); slice <= sliceN; slice++ { + for slice := uint64(0); slice <= sliceNs[cmd.Database]; slice++ { logger.Printf("exporting slice: %d", slice) if err := client.ExportCSV(ctx, cmd.Database, cmd.Frame, slice, w); err != nil { return err diff --git a/db.go b/db.go index a28ec9cf7..fb7724388 100644 --- a/db.go +++ b/db.go @@ -18,6 +18,9 @@ type DB struct { // Frames by name. frames map[string]*Frame + // Max Slice on any node in the cluster, according to this node + remoteMaxSlice uint64 + // Profile attribute storage and cache profileAttrStore *AttrStore @@ -27,9 +30,10 @@ type DB struct { // NewDB returns a new instance of DB. func NewDB(path, name string) *DB { return &DB{ - path: path, - name: name, - frames: make(map[string]*Frame), + path: path, + name: name, + frames: make(map[string]*Frame), + remoteMaxSlice: 0, profileAttrStore: NewAttrStore(filepath.Join(path, "data")), @@ -112,12 +116,15 @@ func (db *DB) Close() error { return nil } -// SliceN returns the max slice in the database. +// SliceN returns the max slice in the database according to this node. func (db *DB) SliceN() uint64 { + if db == nil { + return 0 + } db.mu.Lock() defer db.mu.Unlock() - var max uint64 + max := db.remoteMaxSlice for _, f := range db.frames { if slice := f.SliceN(); slice > max { max = slice @@ -234,3 +241,9 @@ func MergeSchemas(a, b []*DBInfo) []*DBInfo { return dbs } + +func (db *DB) SetRemoteMaxSlice(newmax uint64) { + db.mu.Lock() + defer db.mu.Unlock() + db.remoteMaxSlice = newmax +} diff --git a/executor.go b/executor.go index a2fb40fea..5da490208 100644 --- a/executor.go +++ b/executor.go @@ -53,8 +53,7 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices // If slices aren't specified, then include all of them. if len(slices) == 0 { // Round up the number of slices. - sliceN := e.Index.SliceN() - sliceN += (sliceN % uint64(len(e.Cluster.Nodes))) + uint64(len(e.Cluster.Nodes)) + sliceN := e.Index.DB(db).SliceN() // Generate a slices of all slices. slices = make([]uint64, sliceN+1) diff --git a/executor_test.go b/executor_test.go index e7162bf36..bd4870db8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -281,7 +281,7 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) { t.Fatalf("unexpected db: %s", db) } else if query.String() != `Bitmap(id=10, frame=f)` { t.Fatalf("unexpected query: %s", query.String()) - } else if !reflect.DeepEqual(slices, []uint64{0, 2, 4}) { + } else if !reflect.DeepEqual(slices, []uint64{0}) { //TODO: this is incorrect because the calling node doesn't know about slice 2 t.Fatalf("unexpected slices: %+v", slices) } @@ -390,7 +390,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) - } else if !reflect.DeepEqual(slices, []uint64{0, 2, 4, 6}) { + } else if !reflect.DeepEqual(slices, []uint64{0, 2}) { t.Fatalf("unexpected slices: %+v", slices) } diff --git a/handler.go b/handler.go index 4b326703c..d14faf31e 100644 --- a/handler.go +++ b/handler.go @@ -103,7 +103,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "/slices/max": switch r.Method { case "GET": - h.handleGetSliceMax(w, r) + h.handleGetMaxSlices(w, r) default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } @@ -252,11 +252,11 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) error { - sm := h.Index.SliceN() +func (h *Handler) handleGetMaxSlices(w http.ResponseWriter, r *http.Request) error { + ms := h.Index.SliceNs() if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { - pb := &internal.SliceMaxResponse{ - SliceMax: &sm, + pb := &internal.MaxSlicesResponse{ + MaxSlices: ms, } if buf, err := proto.Marshal(pb); err != nil { return err @@ -265,11 +265,13 @@ func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) erro } return nil } - return json.NewEncoder(w).Encode(sliceMaxResponse{SliceMax: sm}) + return json.NewEncoder(w).Encode(sliceMaxResponse{ + MaxSlices: ms, + }) } type sliceMaxResponse struct { - SliceMax uint64 `json:"SliceMax"` + MaxSlices MaxSlices `json:"MaxSlices"` } // handleDeleteDB handles DELETE /db request. @@ -816,14 +818,15 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } // Determine the maximum number of slices. - sliceN, err := client.SliceN(r.Context()) + sliceNs, err := client.SliceNs(r.Context()) if err != nil { http.Error(w, "cannot determine remote slice count: "+err.Error(), http.StatusInternalServerError) return } // Loop over each slice and import it if this node owns it. - for slice := uint64(0); slice <= sliceN; slice++ { + //travis + for slice := uint64(0); slice <= sliceNs[db]; slice++ { // Ignore this slice if we don't own it. if !h.Cluster.OwnsFragment(h.Host, db, slice) { continue diff --git a/handler_test.go b/handler_test.go index 05f2575e5..e89d02f65 100644 --- a/handler_test.go +++ b/handler_test.go @@ -54,6 +54,30 @@ func TestHandler_Schema(t *testing.T) { } } +// Ensure the handler can return the maxslice map. +func TestHandler_MaxSlices(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + idx.MustCreateFragmentIfNotExists("d0", "f0", 1).MustSetBits(30, (1*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d0", "f0", 1).MustSetBits(30, (1*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d0", "f0", 3).MustSetBits(30, (3*SliceWidth)+4) + + idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+1) + idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+2) + idx.MustCreateFragmentIfNotExists("d1", "f1", 0).MustSetBits(40, (0*SliceWidth)+8) + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil)) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{"MaxSlices":{"d0":3,"d1":0}}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } +} + // Ensure the handler can accept URL arguments. func TestHandler_Query_Args_URL(t *testing.T) { h := NewHandler() diff --git a/index.go b/index.go index 2b3f824d7..189919434 100644 --- a/index.go +++ b/index.go @@ -18,8 +18,7 @@ const DefaultCacheFlushInterval = 1 * time.Minute // Index represents a container for fragments. type Index struct { - mu sync.Mutex - remoteMax uint64 + mu sync.Mutex // Databases by name. dbs map[string]*DB @@ -43,9 +42,8 @@ type Index struct { // NewIndex returns a new instance of Index. func NewIndex() *Index { return &Index{ - dbs: make(map[string]*DB), - remoteMax: 0, - closing: make(chan struct{}, 0), + dbs: make(map[string]*DB), + closing: make(chan struct{}, 0), Stats: NopStatsClient, @@ -108,18 +106,16 @@ func (i *Index) Close() error { return nil } -// SliceN returns the highest slice across all frames. -func (i *Index) SliceN() uint64 { - i.mu.Lock() - defer i.mu.Unlock() +// MaxSlices contains the max known slice -by this node- for each DB +type MaxSlices map[string]uint64 - sliceN := i.remoteMax - for _, db := range i.dbs { - if n := db.SliceN(); n > sliceN { - sliceN = n - } +// SliceNs returns MaxSlice map for all databases. +func (i *Index) SliceNs() MaxSlices { + a := make(MaxSlices) + for _, db := range i.DBs() { + a[db.Name()] = db.SliceN() } - return sliceN + return a } // Schema returns schema data for all databases and frames. @@ -266,12 +262,6 @@ func (i *Index) CreateFragmentIfNotExists(db, frame string, slice uint64) (*Frag return f.CreateFragmentIfNotExists(slice) } -func (i *Index) SetMax(newmax uint64) { - i.mu.Lock() - defer i.mu.Unlock() - i.remoteMax = newmax -} - // monitorCacheFlush periodically flushes all fragment caches sequentially. // This is run in a goroutine. func (i *Index) monitorCacheFlush() { @@ -332,8 +322,6 @@ func (s *IndexSyncer) IsClosing() bool { // SyncIndex compares the index on host with the local index and resolves differences. func (s *IndexSyncer) SyncIndex() error { - sliceN := s.Index.SliceN() - // Iterate over schema in sorted order. for _, di := range s.Index.Schema() { // Verify syncer has not closed. @@ -357,7 +345,7 @@ func (s *IndexSyncer) SyncIndex() error { return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err) } - for slice := uint64(0); slice <= sliceN; slice++ { + for slice := uint64(0); slice <= s.Index.DB(di.Name).SliceN(); slice++ { // Ignore slices that this host doesn't own. if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) { continue diff --git a/index_test.go b/index_test.go index af134658c..8144e4f73 100644 --- a/index_test.go +++ b/index_test.go @@ -111,7 +111,8 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } // Set highest slice. - idx0.SetMax(3) + idx0.DB("d").SetRemoteMaxSlice(1) + idx0.DB("y").SetRemoteMaxSlice(3) // Set up syncer. syncer := pilosa.IndexSyncer{ @@ -119,6 +120,7 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { Host: cluster.Nodes[0].Host, Cluster: cluster, } + if err := syncer.SyncIndex(); err != nil { t.Fatal(err) } @@ -139,10 +141,13 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { } f = idx.Fragment("d", "f0", 1) + a := f.Bitmap(9).Bits() + if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) + } if a := f.Bitmap(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) } - f = idx.Fragment("y", "z", 3) if a := f.Bitmap(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) diff --git a/internal/internal.pb.go b/internal/internal.pb.go index c61ad59c7..0adcfb4ca 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -23,7 +23,7 @@ It has these top-level messages: BlockDataRequest BlockDataResponse Cache - SliceMaxResponse + MaxSlicesResponse */ package internal @@ -38,11 +38,13 @@ var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. -const _ = proto.GoGoProtoPackageIsVersion1 +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package type Bitmap struct { - Bits []uint64 `protobuf:"varint,1,rep,name=Bits" json:"Bits,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` + Bits []uint64 `protobuf:"varint,1,rep,name=Bits,json=bits" json:"Bits,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -66,8 +68,8 @@ func (m *Bitmap) GetAttrs() []*Attr { } type Pair struct { - Key *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"` - Count *uint64 `protobuf:"varint,2,req,name=Count" json:"Count,omitempty"` + Key *uint64 `protobuf:"varint,1,req,name=Key,json=key" json:"Key,omitempty"` + Count *uint64 `protobuf:"varint,2,req,name=Count,json=count" json:"Count,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -91,8 +93,8 @@ func (m *Pair) GetCount() uint64 { } type Bit struct { - BitmapID *uint64 `protobuf:"varint,1,req,name=BitmapID" json:"BitmapID,omitempty"` - ProfileID *uint64 `protobuf:"varint,2,req,name=ProfileID" json:"ProfileID,omitempty"` + BitmapID *uint64 `protobuf:"varint,1,req,name=BitmapID,json=bitmapID" json:"BitmapID,omitempty"` + ProfileID *uint64 `protobuf:"varint,2,req,name=ProfileID,json=profileID" json:"ProfileID,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -116,8 +118,8 @@ func (m *Bit) GetProfileID() uint64 { } type Profile struct { - ID *uint64 `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` + ID *uint64 `protobuf:"varint,1,req,name=ID,json=iD" json:"ID,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -141,10 +143,10 @@ func (m *Profile) GetAttrs() []*Attr { } type Attr struct { - Key *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"` - StringValue *string `protobuf:"bytes,2,opt,name=StringValue" json:"StringValue,omitempty"` - UintValue *uint64 `protobuf:"varint,3,opt,name=UintValue" json:"UintValue,omitempty"` - BoolValue *bool `protobuf:"varint,4,opt,name=BoolValue" json:"BoolValue,omitempty"` + Key *string `protobuf:"bytes,1,req,name=Key,json=key" json:"Key,omitempty"` + StringValue *string `protobuf:"bytes,2,opt,name=StringValue,json=stringValue" json:"StringValue,omitempty"` + UintValue *uint64 `protobuf:"varint,3,opt,name=UintValue,json=uintValue" json:"UintValue,omitempty"` + BoolValue *bool `protobuf:"varint,4,opt,name=BoolValue,json=boolValue" json:"BoolValue,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -182,7 +184,7 @@ func (m *Attr) GetBoolValue() bool { } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -199,13 +201,13 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Query *string `protobuf:"bytes,2,req,name=Query" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,3,rep,name=Slices" json:"Slices,omitempty"` - Profiles *bool `protobuf:"varint,4,opt,name=Profiles" json:"Profiles,omitempty"` - Timestamp *int64 `protobuf:"varint,5,opt,name=Timestamp" json:"Timestamp,omitempty"` - Quantum *uint32 `protobuf:"varint,6,opt,name=Quantum" json:"Quantum,omitempty"` - Remote *bool `protobuf:"varint,7,opt,name=Remote" json:"Remote,omitempty"` + DB *string `protobuf:"bytes,1,req,name=DB,json=dB" json:"DB,omitempty"` + Query *string `protobuf:"bytes,2,req,name=Query,json=query" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,3,rep,name=Slices,json=slices" json:"Slices,omitempty"` + Profiles *bool `protobuf:"varint,4,opt,name=Profiles,json=profiles" json:"Profiles,omitempty"` + Timestamp *int64 `protobuf:"varint,5,opt,name=Timestamp,json=timestamp" json:"Timestamp,omitempty"` + Quantum *uint32 `protobuf:"varint,6,opt,name=Quantum,json=quantum" json:"Quantum,omitempty"` + Remote *bool `protobuf:"varint,7,opt,name=Remote,json=remote" json:"Remote,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -264,9 +266,9 @@ func (m *QueryRequest) GetRemote() bool { } type QueryResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` - Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` + Err *string `protobuf:"bytes,1,opt,name=Err,json=err" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results,json=results" json:"Results,omitempty"` + Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles,json=profiles" json:"Profiles,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -297,10 +299,10 @@ func (m *QueryResponse) GetProfiles() []*Profile { } type QueryResult struct { - Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` - N *uint64 `protobuf:"varint,2,opt,name=N" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed *bool `protobuf:"varint,4,opt,name=Changed" json:"Changed,omitempty"` + Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap,json=bitmap" json:"Bitmap,omitempty"` + N *uint64 `protobuf:"varint,2,opt,name=N,json=n" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,json=pairs" json:"Pairs,omitempty"` + Changed *bool `protobuf:"varint,4,opt,name=Changed,json=changed" json:"Changed,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -338,11 +340,11 @@ func (m *QueryResult) GetChanged() bool { } type ImportRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Frame *string `protobuf:"bytes,2,req,name=Frame" json:"Frame,omitempty"` - Slice *uint64 `protobuf:"varint,3,req,name=Slice" json:"Slice,omitempty"` - BitmapIDs []uint64 `protobuf:"varint,4,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,5,rep,name=ProfileIDs" json:"ProfileIDs,omitempty"` + DB *string `protobuf:"bytes,1,req,name=DB,json=dB" json:"DB,omitempty"` + Frame *string `protobuf:"bytes,2,req,name=Frame,json=frame" json:"Frame,omitempty"` + Slice *uint64 `protobuf:"varint,3,req,name=Slice,json=slice" json:"Slice,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,4,rep,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,5,rep,name=ProfileIDs,json=profileIDs" json:"ProfileIDs,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -387,7 +389,7 @@ func (m *ImportRequest) GetProfileIDs() []uint64 { } type ImportResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"` + Err *string `protobuf:"bytes,1,opt,name=Err,json=err" json:"Err,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -404,10 +406,10 @@ func (m *ImportResponse) GetErr() string { } type BlockDataRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Frame *string `protobuf:"bytes,2,req,name=Frame" json:"Frame,omitempty"` - Slice *uint64 `protobuf:"varint,3,req,name=Slice" json:"Slice,omitempty"` - Block *uint64 `protobuf:"varint,4,req,name=Block" json:"Block,omitempty"` + DB *string `protobuf:"bytes,1,req,name=DB,json=dB" json:"DB,omitempty"` + Frame *string `protobuf:"bytes,2,req,name=Frame,json=frame" json:"Frame,omitempty"` + Slice *uint64 `protobuf:"varint,3,req,name=Slice,json=slice" json:"Slice,omitempty"` + Block *uint64 `protobuf:"varint,4,req,name=Block,json=block" json:"Block,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -445,8 +447,8 @@ func (m *BlockDataRequest) GetBlock() uint64 { } type BlockDataResponse struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,2,rep,name=ProfileIDs" json:"ProfileIDs,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,2,rep,name=ProfileIDs,json=profileIDs" json:"ProfileIDs,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -470,7 +472,7 @@ func (m *BlockDataResponse) GetProfileIDs() []uint64 { } type Cache struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` XXX_unrecognized []byte `json:"-"` } @@ -486,21 +488,21 @@ func (m *Cache) GetBitmapIDs() []uint64 { return nil } -type SliceMaxResponse struct { - SliceMax *uint64 `protobuf:"varint,1,req,name=SliceMax" json:"SliceMax,omitempty"` - XXX_unrecognized []byte `json:"-"` +type MaxSlicesResponse struct { + MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices,json=maxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` + XXX_unrecognized []byte `json:"-"` } -func (m *SliceMaxResponse) Reset() { *m = SliceMaxResponse{} } -func (m *SliceMaxResponse) String() string { return proto.CompactTextString(m) } -func (*SliceMaxResponse) ProtoMessage() {} -func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } +func (m *MaxSlicesResponse) Reset() { *m = MaxSlicesResponse{} } +func (m *MaxSlicesResponse) String() string { return proto.CompactTextString(m) } +func (*MaxSlicesResponse) ProtoMessage() {} +func (*MaxSlicesResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } -func (m *SliceMaxResponse) GetSliceMax() uint64 { - if m != nil && m.SliceMax != nil { - return *m.SliceMax +func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 { + if m != nil { + return m.MaxSlices } - return 0 + return nil } func init() { @@ -518,41 +520,53 @@ func init() { proto.RegisterType((*BlockDataRequest)(nil), "internal.BlockDataRequest") proto.RegisterType((*BlockDataResponse)(nil), "internal.BlockDataResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") - proto.RegisterType((*SliceMaxResponse)(nil), "internal.SliceMaxResponse") + proto.RegisterType((*MaxSlicesResponse)(nil), "internal.MaxSlicesResponse") } +func init() { proto.RegisterFile("internal/internal.proto", fileDescriptorInternal) } + var fileDescriptorInternal = []byte{ - // 499 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x6b, 0xd4, 0x40, - 0x14, 0x65, 0x37, 0xd9, 0xcd, 0xee, 0x4d, 0x77, 0xdd, 0x1d, 0x11, 0x83, 0x50, 0x28, 0x51, 0xa4, - 0xf8, 0x50, 0xa1, 0xe8, 0x8b, 0x6f, 0x6e, 0x57, 0xa1, 0x94, 0x96, 0x7e, 0xa8, 0xcf, 0x0e, 0xeb, - 0xd8, 0x46, 0x93, 0x4c, 0x9c, 0x4c, 0xc0, 0x3e, 0xf9, 0xd7, 0x3d, 0x33, 0x99, 0xc9, 0x46, 0x58, - 0x11, 0x9f, 0x92, 0x7b, 0xe6, 0x9e, 0x7b, 0xce, 0x9c, 0xdc, 0xd0, 0xe3, 0xac, 0xd4, 0x42, 0x95, - 0x3c, 0x7f, 0xe9, 0x5f, 0x8e, 0x2a, 0x25, 0xb5, 0x64, 0x13, 0x5f, 0xa7, 0xaf, 0x69, 0xbc, 0xca, - 0x74, 0xc1, 0x2b, 0xb6, 0x47, 0x21, 0xde, 0xea, 0x64, 0x70, 0x10, 0x1c, 0x86, 0x6c, 0x9f, 0x46, - 0x6f, 0xb5, 0x56, 0x75, 0x32, 0x44, 0x19, 0x1f, 0xcf, 0x8f, 0xba, 0x09, 0x06, 0x4e, 0x53, 0x0a, - 0x2f, 0x79, 0xa6, 0x58, 0x4c, 0xc1, 0x99, 0xb8, 0x07, 0x67, 0x08, 0xce, 0x8c, 0x46, 0x27, 0xb2, - 0x29, 0x35, 0x38, 0x28, 0xd3, 0x17, 0x14, 0x60, 0x20, 0x5b, 0xd0, 0xa4, 0x55, 0x38, 0x5d, 0xbb, - 0xbe, 0x25, 0x4d, 0x2f, 0x95, 0xfc, 0x9a, 0xe5, 0x02, 0x50, 0xdb, 0xfb, 0x8a, 0x22, 0x07, 0x31, - 0xa2, 0x61, 0xd7, 0xf9, 0x0f, 0x17, 0x17, 0x14, 0x9a, 0x67, 0xdf, 0xc5, 0x94, 0x3d, 0xa4, 0xf8, - 0x46, 0xab, 0xac, 0xbc, 0xfd, 0xc4, 0xf3, 0x46, 0x80, 0x39, 0x00, 0x08, 0xc9, 0x8f, 0xe0, 0xb6, - 0x50, 0x00, 0xc8, 0xba, 0x58, 0x49, 0x99, 0xb7, 0x50, 0x08, 0x68, 0x92, 0x1e, 0x52, 0x64, 0xe6, - 0x9d, 0x23, 0x8d, 0x4e, 0x79, 0xb0, 0x53, 0xf9, 0x17, 0xed, 0x5d, 0x35, 0x42, 0xdd, 0x5f, 0x8b, - 0x1f, 0x8d, 0xa8, 0xb5, 0x31, 0xbd, 0x5e, 0x39, 0x03, 0x88, 0xc1, 0x9e, 0xd9, 0xab, 0x4d, 0xd9, - 0x9c, 0xc6, 0x37, 0x79, 0xb6, 0x11, 0x35, 0x74, 0x4d, 0xb2, 0xc8, 0xc3, 0x5d, 0xb5, 0x6e, 0x65, - 0x8d, 0x93, 0x0f, 0x59, 0x81, 0x31, 0xbc, 0xa8, 0x92, 0x11, 0xa0, 0x80, 0x3d, 0xa0, 0xe8, 0xaa, - 0xe1, 0xa5, 0x6e, 0x8a, 0x64, 0x0c, 0x60, 0x66, 0xa6, 0x5c, 0x8b, 0x42, 0x6a, 0x91, 0x44, 0xd6, - 0x6a, 0x46, 0x33, 0x67, 0xa0, 0xae, 0x64, 0x59, 0x0b, 0x93, 0xc1, 0x3b, 0xa5, 0x60, 0xc1, 0x5c, - 0xf7, 0x39, 0x45, 0x38, 0x68, 0x72, 0xed, 0x93, 0x7b, 0xb4, 0xf5, 0xef, 0x69, 0x38, 0x65, 0x4f, - 0x7b, 0x5e, 0x02, 0xdb, 0xb8, 0xdc, 0x36, 0xba, 0x93, 0xf4, 0x1b, 0xc5, 0x7d, 0xce, 0x81, 0xdf, - 0x18, 0xab, 0x15, 0x1f, 0x2f, 0xb6, 0x0c, 0xb7, 0x49, 0x53, 0x1a, 0x5c, 0xd8, 0xdc, 0xed, 0x07, - 0x34, 0x7b, 0xe2, 0xa7, 0xf7, 0x62, 0xb4, 0xeb, 0x83, 0x6b, 0x9e, 0xdc, 0xf1, 0xf2, 0x56, 0x7c, - 0x71, 0x5f, 0xe0, 0x33, 0xcd, 0x4e, 0x8b, 0x4a, 0x2a, 0xfd, 0x97, 0x60, 0xdf, 0x2b, 0x5e, 0x08, - 0x17, 0x2c, 0x4a, 0x1b, 0x2c, 0x66, 0xbb, 0xad, 0xf2, 0x7b, 0x66, 0x82, 0x35, 0x51, 0x83, 0xdd, - 0x2d, 0x5a, 0x8d, 0x64, 0x81, 0xa5, 0xfb, 0x34, 0xf7, 0x0a, 0x3b, 0x92, 0x4b, 0xcf, 0x68, 0xb1, - 0xca, 0xe5, 0xe6, 0xfb, 0x9a, 0x6b, 0xfe, 0xff, 0x1e, 0x50, 0x5a, 0x36, 0xf4, 0xcd, 0x56, 0xbf, - 0xa1, 0x65, 0x6f, 0x98, 0x93, 0xfb, 0xc3, 0xe7, 0x60, 0x87, 0xcf, 0xa1, 0xf5, 0xf9, 0x04, 0x3f, - 0x13, 0xdf, 0xdc, 0xed, 0xea, 0x4f, 0x9f, 0xd1, 0xc2, 0xaa, 0x9e, 0xf3, 0x9f, 0xdd, 0x58, 0xac, - 0x95, 0xc7, 0xda, 0x9f, 0xe7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x12, 0x50, 0x70, 0xfe, - 0x03, 0x00, 0x00, + // 667 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x54, 0x5f, 0x6b, 0xd4, 0x4e, + 0x14, 0x65, 0xf2, 0x67, 0xb3, 0xb9, 0xdb, 0x2e, 0xed, 0xd0, 0xdf, 0xcf, 0x50, 0x44, 0x42, 0x50, + 0x08, 0x82, 0x2d, 0xf4, 0x49, 0x44, 0x28, 0xa6, 0x5b, 0xb1, 0x48, 0xa5, 0x9d, 0xaa, 0xef, 0xb3, + 0xdb, 0x69, 0x1b, 0x9a, 0x7f, 0x9d, 0x99, 0x14, 0xf7, 0xd1, 0x07, 0x1f, 0xfc, 0x0c, 0x7e, 0x11, + 0x3f, 0x9e, 0xcc, 0x64, 0x26, 0xd9, 0x52, 0xac, 0x3e, 0xf8, 0x96, 0x73, 0x6e, 0xee, 0xdc, 0x73, + 0xef, 0x99, 0x3b, 0xf0, 0x28, 0xaf, 0x24, 0xe3, 0x15, 0x2d, 0x76, 0xed, 0xc7, 0x4e, 0xc3, 0x6b, + 0x59, 0xe3, 0xb1, 0xc5, 0x49, 0x06, 0xa3, 0x2c, 0x97, 0x25, 0x6d, 0x30, 0x06, 0x2f, 0xcb, 0xa5, + 0x88, 0x50, 0xec, 0xa6, 0x1e, 0xf1, 0xe6, 0xb9, 0x14, 0xf8, 0x29, 0xf8, 0x6f, 0xa4, 0xe4, 0x22, + 0x72, 0x62, 0x37, 0x9d, 0xec, 0x4d, 0x77, 0xfa, 0x73, 0x14, 0x4d, 0x7c, 0xaa, 0x82, 0xc9, 0x0e, + 0x78, 0x27, 0x34, 0xe7, 0x78, 0x03, 0xdc, 0xf7, 0x6c, 0x19, 0xa1, 0xd8, 0x49, 0x3d, 0xe2, 0x5e, + 0xb3, 0x25, 0xde, 0x02, 0xff, 0xa0, 0x6e, 0x2b, 0x19, 0x39, 0x9a, 0xf3, 0x17, 0x0a, 0x24, 0xfb, + 0xe0, 0x66, 0xb9, 0xc4, 0xdb, 0x30, 0xee, 0x4a, 0x1f, 0xcd, 0x4c, 0xce, 0x78, 0x6e, 0x30, 0x7e, + 0x0c, 0xe1, 0x09, 0xaf, 0x2f, 0xf2, 0x82, 0x1d, 0xcd, 0x4c, 0x72, 0xd8, 0x58, 0x22, 0xd9, 0x87, + 0xc0, 0x44, 0xf1, 0x14, 0x9c, 0x3e, 0xdd, 0xc9, 0x67, 0x7f, 0xa9, 0xf8, 0x16, 0x3c, 0x05, 0x57, + 0x15, 0x87, 0x9d, 0xe2, 0x18, 0x26, 0x67, 0x92, 0xe7, 0xd5, 0xe5, 0x67, 0x5a, 0xb4, 0x2c, 0x72, + 0x62, 0x94, 0x86, 0x64, 0x22, 0x06, 0x4a, 0x49, 0xfb, 0x94, 0x57, 0xb2, 0x8b, 0xbb, 0x31, 0x52, + 0xd2, 0x5a, 0x4b, 0xa8, 0x68, 0x56, 0xd7, 0x45, 0x17, 0xf5, 0x62, 0x94, 0x8e, 0x49, 0x38, 0xb7, + 0x44, 0xb2, 0x0b, 0x81, 0xaa, 0x7b, 0x4c, 0x9b, 0x41, 0x28, 0x7a, 0x48, 0xe8, 0x4f, 0x04, 0x6b, + 0xa7, 0x2d, 0xe3, 0x4b, 0xc2, 0x6e, 0x5a, 0x26, 0xa4, 0xea, 0x77, 0x96, 0x19, 0xc1, 0xce, 0x79, + 0xa6, 0x26, 0xac, 0xe3, 0x7a, 0x48, 0x21, 0xf1, 0x6f, 0x14, 0xc0, 0xff, 0xc3, 0xe8, 0xac, 0xc8, + 0x17, 0x4c, 0x44, 0xae, 0x76, 0x73, 0x24, 0x34, 0x52, 0x23, 0x37, 0x83, 0x13, 0x46, 0xdc, 0xd8, + 0x4c, 0x55, 0x28, 0xe5, 0x1f, 0xf3, 0x92, 0x09, 0x49, 0xcb, 0x26, 0xf2, 0x63, 0x94, 0xba, 0x24, + 0x94, 0x96, 0xc0, 0x11, 0x04, 0xa7, 0x2d, 0xad, 0x64, 0x5b, 0x46, 0xa3, 0x18, 0xa5, 0xeb, 0x24, + 0xb8, 0xe9, 0xa0, 0xaa, 0x45, 0x58, 0x59, 0x4b, 0x16, 0x05, 0xfa, 0xc4, 0x11, 0xd7, 0x28, 0xf9, + 0x8a, 0x60, 0xdd, 0x48, 0x17, 0x4d, 0x5d, 0x09, 0xa6, 0xa6, 0x7d, 0xc8, 0x79, 0x84, 0xf4, 0x4c, + 0x5d, 0xc6, 0x39, 0xde, 0x85, 0x80, 0x30, 0xd1, 0x16, 0xd2, 0xfa, 0xf5, 0xdf, 0x30, 0x06, 0x9b, + 0xdb, 0x16, 0x92, 0x04, 0xbc, 0xfb, 0x0b, 0xbf, 0x58, 0x69, 0xc0, 0xd5, 0x19, 0x9b, 0x43, 0x86, + 0x89, 0x0c, 0x3d, 0x25, 0xdf, 0x10, 0x4c, 0x56, 0xce, 0xc1, 0xa9, 0xbd, 0xed, 0x5a, 0xc4, 0x64, + 0x6f, 0x63, 0x48, 0xee, 0x78, 0x32, 0xea, 0xae, 0x20, 0x5e, 0x03, 0xf4, 0x41, 0xbb, 0xef, 0x11, + 0x54, 0x29, 0xb3, 0xd4, 0x0d, 0xb7, 0x35, 0x57, 0xcc, 0x52, 0x34, 0xf1, 0x1b, 0x15, 0x54, 0x33, + 0x3a, 0xb8, 0xa2, 0xd5, 0x25, 0x3b, 0x37, 0xc3, 0x0d, 0x16, 0x1d, 0x4c, 0xbe, 0x23, 0x58, 0x3f, + 0x2a, 0x9b, 0x9a, 0xcb, 0x07, 0x7c, 0x7c, 0xcb, 0x69, 0xc9, 0xac, 0x8f, 0x17, 0x0a, 0x28, 0x56, + 0xfb, 0x18, 0xb9, 0xdd, 0xfe, 0x68, 0x1b, 0xf5, 0x1d, 0x33, 0x8b, 0xa2, 0x6c, 0x54, 0x06, 0x87, + 0x76, 0x73, 0x04, 0x7e, 0x02, 0xd0, 0xaf, 0x8e, 0x88, 0x7c, 0x1d, 0x86, 0x7e, 0x77, 0x44, 0x92, + 0xc0, 0xd4, 0x4a, 0xf9, 0x9d, 0x2f, 0xc9, 0x39, 0x6c, 0x64, 0x45, 0xbd, 0xb8, 0x9e, 0x51, 0x49, + 0xff, 0x85, 0xe2, 0x2d, 0xf0, 0xf5, 0x79, 0x91, 0xd7, 0xb1, 0x73, 0x05, 0x92, 0x53, 0xd8, 0x5c, + 0xa9, 0x62, 0xc4, 0xdc, 0x69, 0x0e, 0x3d, 0xdc, 0x9c, 0x73, 0xaf, 0xb9, 0x67, 0xe0, 0x1f, 0xd0, + 0xc5, 0xd5, 0x1f, 0x8e, 0x49, 0x7e, 0x20, 0xd8, 0x3c, 0xa6, 0x5f, 0xba, 0x1d, 0xe9, 0x4b, 0xbf, + 0x83, 0xb0, 0x27, 0xcd, 0x5a, 0x3e, 0x1f, 0x9c, 0xbe, 0xf7, 0xff, 0xc0, 0x1c, 0x56, 0x92, 0x2f, + 0x49, 0x58, 0x5a, 0xbc, 0xfd, 0x1a, 0xa6, 0x77, 0x83, 0x6a, 0xc6, 0xd7, 0xfa, 0xa5, 0x41, 0xf6, + 0xa5, 0xd9, 0x02, 0xff, 0xb6, 0x7f, 0x63, 0x3c, 0xd2, 0x81, 0x57, 0xce, 0x4b, 0xf4, 0x2b, 0x00, + 0x00, 0xff, 0xff, 0x1a, 0xd6, 0x56, 0x91, 0xb7, 0x05, 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index 79e1d627d..6c2990360 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -1,3 +1,4 @@ +syntax = "proto2"; package internal; message Bitmap { @@ -82,6 +83,6 @@ message Cache { repeated uint64 BitmapIDs = 1; } -message SliceMaxResponse { - required uint64 SliceMax = 1; +message MaxSlicesResponse { + map MaxSlices = 1; } diff --git a/server.go b/server.go index 04853a03d..ffde80760 100644 --- a/server.go +++ b/server.go @@ -21,7 +21,8 @@ import ( // Default server settings. const ( DefaultAntiEntropyInterval = 10 * time.Minute - DefaultPollingInterval = 60 * time.Second + //DefaultPollingInterval = 60 * time.Second + DefaultPollingInterval = 5 * time.Second ) // Server represents an index wrapped by a running HTTP server. @@ -115,7 +116,7 @@ func (s *Server) Open() error { // Start background monitoring. s.wg.Add(2) go func() { defer s.wg.Done(); s.monitorAntiEntropy() }() - go func() { defer s.wg.Done(); s.monitorMaxSlice() }() + go func() { defer s.wg.Done(); s.monitorMaxSlices() }() return nil } @@ -180,14 +181,14 @@ func (s *Server) monitorAntiEntropy() { } } -// monitorMaxSlice periodically pulls the highest slice from each node in the cluster. -func (s *Server) monitorMaxSlice() { +// monitorMaxSlices periodically pulls the highest slice from each node in the cluster. +func (s *Server) monitorMaxSlices() { // Ignore if only one node in the cluster. if len(s.Cluster.Nodes) <= 1 { return } - ticker := time.NewTicker(time.Second * time.Duration(s.PollingInterval)) + ticker := time.NewTicker(s.PollingInterval) defer ticker.Stop() for { @@ -197,24 +198,27 @@ func (s *Server) monitorMaxSlice() { case <-ticker.C: } - oldmax := s.Index.SliceN() - newmax := oldmax + oldmaxslices := s.Index.SliceNs() for _, node := range s.Cluster.Nodes { if s.Host != node.Host { - newslice, _ := checkMaxSlice(node.Host) - if newslice > newmax { - newmax = newslice + maxSlices, _ := checkMaxSlices(node.Host) + for db, newmax := range maxSlices { + // we're not going to create a db locally if we don't know about it already + // TODO: consider changing this so we DO create a db locally + // do we want/need nodes to have empty files structures? + if localdb := s.Index.DB(db); localdb != nil { + if newmax > oldmaxslices[db] { + oldmaxslices[db] = newmax + localdb.SetRemoteMaxSlice(newmax) + } + } } } } - - if newmax > oldmax { - s.Index.SetMax(newmax) - } } } -func checkMaxSlice(hostport string) (uint64, error) { +func checkMaxSlices(hostport string) (MaxSlices, error) { // Create HTTP request. req, err := http.NewRequest("GET", (&url.URL{ Scheme: "http", @@ -223,7 +227,7 @@ func checkMaxSlice(hostport string) (uint64, error) { }).String(), nil) if err != nil { - return 0, err + return nil, err } // Require protobuf encoding. @@ -233,28 +237,28 @@ func checkMaxSlice(hostport string) (uint64, error) { // Send request to remote node. resp, err := http.DefaultClient.Do(req) if err != nil { - return 0, err + return nil, err } defer resp.Body.Close() // Read response into buffer. body, err := ioutil.ReadAll(resp.Body) if err != nil { - return 0, err + return nil, err } // Check status code. if resp.StatusCode != http.StatusOK { - return 0, fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body) + return nil, fmt.Errorf("invalid status: code=%d, err=%s", resp.StatusCode, body) } // Decode response object. - pb := internal.SliceMaxResponse{} + pb := internal.MaxSlicesResponse{} if err = proto.Unmarshal(body, &pb); err != nil { - return 0, err + return nil, err } - return *pb.SliceMax, nil + return pb.MaxSlices, nil } From b0b0de5703bb433dc68d2e897bc0d6969c24dab2 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 21 Nov 2016 16:25:15 -0600 Subject: [PATCH 02/11] fixes a minor logic bug in the cluster tests --- cluster_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cluster_test.go b/cluster_test.go index 7b47ff2af..9ec7ead82 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -41,7 +41,7 @@ func TestCluster_Partition(t *testing.T) { c.PartitionN = partitionN partitionID := c.Partition(db, slice) - if partitionID < 0 || partitionID > partitionN { + if partitionID < 0 || partitionID >= partitionN { t.Errorf("partition out of range: slice=%d, p=%d, n=%d", slice, partitionID, partitionN) } From 93e3590737835e25d2bb94831777e84eff896526 Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 21 Nov 2016 16:36:17 -0600 Subject: [PATCH 03/11] set DefaultPollingInterval back to its original value --- server.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/server.go b/server.go index ffde80760..d9399926d 100644 --- a/server.go +++ b/server.go @@ -21,8 +21,7 @@ import ( // Default server settings. const ( DefaultAntiEntropyInterval = 10 * time.Minute - //DefaultPollingInterval = 60 * time.Second - DefaultPollingInterval = 5 * time.Second + DefaultPollingInterval = 60 * time.Second ) // Server represents an index wrapped by a running HTTP server. From dd6f41f5512d6b8872d0c327423a2822941814e8 Mon Sep 17 00:00:00 2001 From: Travis Date: Wed, 30 Nov 2016 12:59:17 -0600 Subject: [PATCH 04/11] standardizes on MaxSlice (instead of SliceN) creates DB locally even if node doesn't have data for that DB --- client.go | 8 ++++---- cmd/pilosactl/main.go | 4 ++-- db.go | 6 +++--- executor.go | 16 ++++++++-------- frame.go | 8 ++++---- handler.go | 12 ++++++------ index.go | 13 +++++-------- server.go | 17 ++++++++++++----- 8 files changed, 44 insertions(+), 40 deletions(-) diff --git a/client.go b/client.go index 8fe8ee7d8..ed6a1a2ab 100644 --- a/client.go +++ b/client.go @@ -45,8 +45,8 @@ func NewClient(host string) (*Client, error) { // Host returns the host the client was initialized with. func (c *Client) Host() string { return c.host } -// SliceNs returns the number of slices on a server by database. -func (c *Client) SliceNs(ctx context.Context) (MaxSlices, error) { +// MaxSliceByDatabase returns the number of slices on a server by database. +func (c *Client) MaxSliceByDatabase(ctx context.Context) (map[string]uint64, error) { // Execute request against the host. u := url.URL{ Scheme: "http", @@ -362,13 +362,13 @@ func (c *Client) BackupTo(ctx context.Context, w io.Writer, db, frame string) er tw := tar.NewWriter(w) // Find the maximum number of slices. - sliceNs, err := c.SliceNs(ctx) + maxSlices, err := c.MaxSliceByDatabase(ctx) if err != nil { return fmt.Errorf("slice n: %s", err) } // Backup every slice to the tar file. - for i := uint64(0); i <= sliceNs[db]; i++ { + for i := uint64(0); i <= maxSlices[db]; i++ { if err := c.backupSliceTo(ctx, tw, db, frame, i); err != nil { return err } diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index b7ab3247f..88b4c328f 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -480,13 +480,13 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } // Determine slice count. - sliceNs, err := client.SliceNs(ctx) + maxSlices, err := client.MaxSliceByDatabase(ctx) if err != nil { return err } // Export each slice. - for slice := uint64(0); slice <= sliceNs[cmd.Database]; slice++ { + for slice := uint64(0); slice <= maxSlices[cmd.Database]; slice++ { logger.Printf("exporting slice: %d", slice) if err := client.ExportCSV(ctx, cmd.Database, cmd.Frame, slice, w); err != nil { return err diff --git a/db.go b/db.go index fb7724388..056214f7d 100644 --- a/db.go +++ b/db.go @@ -116,8 +116,8 @@ func (db *DB) Close() error { return nil } -// SliceN returns the max slice in the database according to this node. -func (db *DB) SliceN() uint64 { +// MaxSlice returns the max slice in the database according to this node. +func (db *DB) MaxSlice() uint64 { if db == nil { return 0 } @@ -126,7 +126,7 @@ func (db *DB) SliceN() uint64 { max := db.remoteMaxSlice for _, f := range db.frames { - if slice := f.SliceN(); slice > max { + if slice := f.MaxSlice(); slice > max { max = slice } } diff --git a/executor.go b/executor.go index f966a78ec..3ef806940 100644 --- a/executor.go +++ b/executor.go @@ -53,10 +53,10 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices // If slices aren't specified, then include all of them. if len(slices) == 0 { // Round up the number of slices. - sliceN := e.Index.DB(db).SliceN() + maxSlice := e.Index.DB(db).MaxSlice() // Generate a slices of all slices. - slices = make([]uint64, sliceN+1) + slices = make([]uint64, maxSlice+1) for i := range slices { slices[i] = uint64(i) } @@ -713,7 +713,7 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c // Iterate over all map responses and reduce. var result interface{} - var sliceN int + var maxSlice int for { select { case <-ctx.Done(): @@ -738,8 +738,8 @@ func (e *Executor) mapReduce(ctx context.Context, db string, slices []uint64, c result = reduceFn(result, resp.result) // If all slices have been processed then return. - sliceN += len(resp.slices) - if sliceN >= len(slices) { + maxSlice += len(resp.slices) + if maxSlice >= len(slices) { return result, nil } } @@ -797,7 +797,7 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu } // Reduce results - var sliceN int + var maxSlice int var result interface{} for { select { @@ -808,11 +808,11 @@ func (e *Executor) mapperLocal(ctx context.Context, slices []uint64, mapFn mapFu return nil, resp.err } result = reduceFn(result, resp.result) - sliceN++ + maxSlice++ } // Exit once all slices are processed. - if sliceN == len(slices) { + if maxSlice == len(slices) { return result, nil } } diff --git a/frame.go b/frame.go index 56e4b3f1a..c8f18f8a1 100644 --- a/frame.go +++ b/frame.go @@ -58,8 +58,8 @@ func (f *Frame) Path() string { return f.path } // BitmapAttrStore returns the attribute storage. func (f *Frame) BitmapAttrStore() *AttrStore { return f.bitmapAttrStore } -// SliceN returns the max slice in the frame. -func (f *Frame) SliceN() uint64 { +// MaxSlice returns the max slice in the frame. +func (f *Frame) MaxSlice() uint64 { f.mu.Lock() defer f.mu.Unlock() @@ -128,7 +128,7 @@ func (f *Frame) openFragments() error { frag.BitmapAttrStore = f.bitmapAttrStore f.fragments[frag.Slice()] = frag - f.stats.Count("sliceN", 1) + f.stats.Count("maxSlice", 1) } return nil @@ -202,7 +202,7 @@ func (f *Frame) createFragmentIfNotExists(slice uint64) (*Fragment, error) { // Save to lookup. f.fragments[slice] = frag - f.stats.Count("sliceN", 1) + f.stats.Count("maxSlice", 1) return frag, nil } diff --git a/handler.go b/handler.go index 153548414..c278ddbd7 100644 --- a/handler.go +++ b/handler.go @@ -103,7 +103,7 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { case "/slices/max": switch r.Method { case "GET": - h.handleGetMaxSlices(w, r) + h.handleGetSliceMax(w, r) default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } @@ -252,8 +252,8 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { } } -func (h *Handler) handleGetMaxSlices(w http.ResponseWriter, r *http.Request) error { - ms := h.Index.SliceNs() +func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) error { + ms := h.Index.MaxSlices() if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { pb := &internal.MaxSlicesResponse{ MaxSlices: ms, @@ -271,7 +271,7 @@ func (h *Handler) handleGetMaxSlices(w http.ResponseWriter, r *http.Request) err } type sliceMaxResponse struct { - MaxSlices MaxSlices `json:"MaxSlices"` + MaxSlices map[string]uint64 `json:"MaxSlices"` } // handleDeleteDB handles DELETE /db request. @@ -818,7 +818,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } // Determine the maximum number of slices. - sliceNs, err := client.SliceNs(r.Context()) + maxSlices, err := client.MaxSliceByDatabase(r.Context()) if err != nil { http.Error(w, "cannot determine remote slice count: "+err.Error(), http.StatusInternalServerError) return @@ -826,7 +826,7 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) // Loop over each slice and import it if this node owns it. //travis - for slice := uint64(0); slice <= sliceNs[db]; slice++ { + for slice := uint64(0); slice <= maxSlices[db]; slice++ { // Ignore this slice if we don't own it. if !h.Cluster.OwnsFragment(h.Host, db, slice) { continue diff --git a/index.go b/index.go index 189919434..6b5de11d5 100644 --- a/index.go +++ b/index.go @@ -106,14 +106,11 @@ func (i *Index) Close() error { return nil } -// MaxSlices contains the max known slice -by this node- for each DB -type MaxSlices map[string]uint64 - -// SliceNs returns MaxSlice map for all databases. -func (i *Index) SliceNs() MaxSlices { - a := make(MaxSlices) +// MaxSlices returns MaxSlice map for all databases. +func (i *Index) MaxSlices() map[string]uint64 { + a := make(map[string]uint64) for _, db := range i.DBs() { - a[db.Name()] = db.SliceN() + a[db.Name()] = db.MaxSlice() } return a } @@ -345,7 +342,7 @@ func (s *IndexSyncer) SyncIndex() error { return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err) } - for slice := uint64(0); slice <= s.Index.DB(di.Name).SliceN(); slice++ { + for slice := uint64(0); slice <= s.Index.DB(di.Name).MaxSlice(); slice++ { // Ignore slices that this host doesn't own. if !s.Cluster.OwnsFragment(s.Host, di.Name, slice) { continue diff --git a/server.go b/server.go index b9f0a38b7..7323935fe 100644 --- a/server.go +++ b/server.go @@ -197,19 +197,26 @@ func (s *Server) monitorMaxSlices() { case <-ticker.C: } - oldmaxslices := s.Index.SliceNs() + oldmaxslices := s.Index.MaxSlices() for _, node := range s.Cluster.Nodes { if s.Host != node.Host { maxSlices, _ := checkMaxSlices(node.Host) for db, newmax := range maxSlices { - // we're not going to create a db locally if we don't know about it already - // TODO: consider changing this so we DO create a db locally - // do we want/need nodes to have empty files structures? + // if we don't know about a db locally, create it + // so that the /schema endpoint can report it if localdb := s.Index.DB(db); localdb != nil { if newmax > oldmaxslices[db] { oldmaxslices[db] = newmax localdb.SetRemoteMaxSlice(newmax) } + } else { + d, err := s.Index.CreateDBIfNotExists(db) + if err != nil { + s.logger().Printf("Failed to create DB locally: %s", db) + return + } + oldmaxslices[db] = newmax + d.SetRemoteMaxSlice(newmax) } } } @@ -217,7 +224,7 @@ func (s *Server) monitorMaxSlices() { } } -func checkMaxSlices(hostport string) (MaxSlices, error) { +func checkMaxSlices(hostport string) (map[string]uint64, error) { // Create HTTP request. req, err := http.NewRequest("GET", (&url.URL{ Scheme: "http", From c6e2b1294b3e2cac05d6f791b2e4e00eb4b2708a Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 8 Dec 2016 12:29:49 -0700 Subject: [PATCH 05/11] Add time-based frames and Range() support. --- attr.go | 40 +- bitmap.go | 4 +- cache.go | 9 +- client.go | 24 +- client_test.go | 6 +- db.go | 149 ++ db_test.go | 68 + executor.go | 52 +- executor_test.go | 72 +- fragment.go | 44 +- fragment_test.go | 46 +- frame.go | 97 +- frame_test.go | 35 + handler.go | 184 +- handler_test.go | 101 +- index_test.go | 44 +- internal/internal.go | 2 + internal/internal.pb.go | 3847 +++++++++++++++++++++++++++++++++++---- internal/internal.proto | 84 +- pilosa.go | 13 +- server.go | 3 +- time.go | 228 +-- time_test.go | 242 +-- 23 files changed, 4561 insertions(+), 833 deletions(-) diff --git a/attr.go b/attr.go index dacbf9bbb..1425676cf 100644 --- a/attr.go +++ b/attr.go @@ -17,6 +17,13 @@ import ( // AttrBlockSize is the size of attribute blocks for anti-entropy. const AttrBlockSize = 100 +// Attribute data type enum. +const ( + AttrTypeString = 1 + AttrTypeUint = 2 + AttrTypeBool = 3 +) + // AttrStore represents a storage layer for attributes. type AttrStore struct { mu sync.Mutex @@ -303,32 +310,39 @@ func decodeAttrs(pb []*internal.Attr) map[string]interface{} { // encodeAttr converts a key/value pair into an Attr internal representation. func encodeAttr(key string, value interface{}) *internal.Attr { - pb := &internal.Attr{Key: proto.String(key)} + pb := &internal.Attr{Key: key} switch value := value.(type) { case string: - pb.StringValue = proto.String(value) + pb.Type = AttrTypeString + pb.StringValue = value case float64: - pb.UintValue = proto.Uint64(uint64(value)) + pb.Type = AttrTypeUint + pb.UintValue = uint64(value) case uint64: - pb.UintValue = proto.Uint64(value) + pb.Type = AttrTypeUint + pb.UintValue = value case int64: - pb.UintValue = proto.Uint64(uint64(value)) + pb.Type = AttrTypeUint + pb.UintValue = uint64(value) case bool: - pb.BoolValue = proto.Bool(value) + pb.Type = AttrTypeBool + pb.BoolValue = value } return pb } // decodeAttr converts from an Attr internal representation to a key/value pair. func decodeAttr(attr *internal.Attr) (key string, value interface{}) { - if attr.StringValue != nil { - return attr.GetKey(), attr.GetStringValue() - } else if attr.UintValue != nil { - return attr.GetKey(), attr.GetUintValue() - } else if attr.BoolValue != nil { - return attr.GetKey(), attr.GetBoolValue() + switch attr.Type { + case AttrTypeString: + return attr.Key, attr.StringValue + case AttrTypeUint: + return attr.Key, attr.UintValue + case AttrTypeBool: + return attr.Key, attr.BoolValue + default: + return attr.Key, nil } - return attr.GetKey(), nil } // cloneAttrs returns a shallow clone of m. diff --git a/bitmap.go b/bitmap.go index b3d8d0096..55664b24b 100644 --- a/bitmap.go +++ b/bitmap.go @@ -227,8 +227,8 @@ func decodeBitmap(pb *internal.Bitmap) *Bitmap { } b := NewBitmap() - b.Attrs = decodeAttrs(pb.GetAttrs()) - for _, v := range pb.GetBits() { + b.Attrs = decodeAttrs(pb.Attrs) + for _, v := range pb.Bits { b.SetBit(v) } return b diff --git a/cache.go b/cache.go index 4efce34f0..a1cba3dec 100644 --- a/cache.go +++ b/cache.go @@ -7,7 +7,6 @@ import ( "sort" "time" - "github.com/gogo/protobuf/proto" "github.com/golang/groupcache/lru" "github.com/pilosa/pilosa/internal" ) @@ -216,15 +215,15 @@ type Pair struct { func encodePair(p Pair) *internal.Pair { return &internal.Pair{ - Key: proto.Uint64(p.Key), - Count: proto.Uint64(p.Count), + Key: p.Key, + Count: p.Count, } } func decodePair(pb *internal.Pair) Pair { return Pair{ - Key: pb.GetKey(), - Count: pb.GetCount(), + Key: pb.Key, + Count: pb.Count, } } diff --git a/client.go b/client.go index d9796700d..9c13fe2ff 100644 --- a/client.go +++ b/client.go @@ -151,9 +151,9 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire // Encode query request. buf, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String(db), - Query: proto.String(query), - Remote: proto.Bool(!allowRedirect), + DB: db, + Query: query, + Remote: !allowRedirect, }) if err != nil { return nil, fmt.Errorf("marshal: %s", err) @@ -187,7 +187,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire var qresp internal.QueryResponse if err := proto.Unmarshal(body, &qresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.GetErr(); s != "" { + } else if s := qresp.Err; s != "" { return nil, errors.New(s) } @@ -230,9 +230,9 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e // Marshal bits to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ - DB: proto.String(db), - Frame: proto.String(frame), - Slice: proto.Uint64(slice), + DB: db, + Frame: frame, + Slice: slice, BitmapIDs: bitmapIDs, ProfileIDs: profileIDs, }) @@ -272,7 +272,7 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { var isresp internal.ImportResponse if err := proto.Unmarshal(body, &isresp); err != nil { return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.GetErr(); s != "" { + } else if s := isresp.Err; s != "" { return errors.New(s) } @@ -644,10 +644,10 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uin // BlockData returns bitmap/profile id pairs for a block. func (c *Client) BlockData(ctx context.Context, db, frame string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ - DB: proto.String(db), - Frame: proto.String(frame), - Slice: proto.Uint64(slice), - Block: proto.Uint64(uint64(block)), + DB: db, + Frame: frame, + Slice: slice, + Block: uint64(block), }) if err != nil { return nil, nil, err diff --git a/client_test.go b/client_test.go index d81e42a72..f37fea6ee 100644 --- a/client_test.go +++ b/client_test.go @@ -96,11 +96,11 @@ func TestClient_FragmentBlocks(t *testing.T) { defer idx.Close() // Set two bits on blocks 0 & 3. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(pilosa.HashBlockSize*3, 100, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, 1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, 1) s := NewServer() defer s.Close() diff --git a/db.go b/db.go index a28ec9cf7..f4e88333a 100644 --- a/db.go +++ b/db.go @@ -3,10 +3,15 @@ package pilosa import ( "errors" "fmt" + "io/ioutil" "os" "path/filepath" "sort" "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) // DB represents a container for frames. @@ -15,6 +20,10 @@ type DB struct { path string name string + // Default time quantum for all frames in database. + // This can be overridden by individual frames. + timeQuantum TimeQuantum + // Frames by name. frames map[string]*Frame @@ -53,6 +62,11 @@ func (db *DB) Open() error { return err } + // Read meta file. + if err := db.loadMeta(); err != nil { + return err + } + if err := db.openFrames(); err != nil { return err } @@ -93,6 +107,45 @@ func (db *DB) openFrames() error { return nil } +// loadMeta reads meta data for the database, if any. +func (db *DB) loadMeta() error { + var pb internal.DB + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(db.path, "meta")) + if os.IsNotExist(err) { + db.timeQuantum = "" + return nil + } else if err != nil { + return err + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + } + + // Copy metadata fields. + db.timeQuantum = TimeQuantum(pb.TimeQuantum) + + return nil +} + +// saveMeta writes meta data for the database. +func (db *DB) saveMeta() error { + // Marshal metadata. + buf, err := proto.Marshal(&internal.DB{TimeQuantum: string(db.timeQuantum)}) + if err != nil { + return err + } + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(db.path, "meta"), buf, 0666); err != nil { + return err + } + + return nil +} + // Close closes the database and its frames. func (db *DB) Close() error { db.mu.Lock() @@ -126,6 +179,34 @@ func (db *DB) SliceN() uint64 { return max } +// TimeQuantum returns the default time quantum for the database. +func (db *DB) TimeQuantum() TimeQuantum { + db.mu.Lock() + defer db.mu.Unlock() + return db.timeQuantum +} + +// SetTimeQuantum sets the default time quantum for the database. +func (db *DB) SetTimeQuantum(q TimeQuantum) error { + db.mu.Lock() + defer db.mu.Unlock() + + // Validate input. + if !q.Valid() { + return ErrInvalidTimeQuantum + } + + // Update value on database. + db.timeQuantum = q + + // Perist meta data to disk. + if err := db.saveMeta(); err != nil { + return err + } + + return nil +} + // FramePath returns the path to a frame in the database. func (db *DB) FramePath(name string) string { return filepath.Join(db.path, name) } @@ -187,6 +268,74 @@ func (db *DB) newFrame(path, name string) *Frame { return f } +// DeleteFrame removes a frame from the database. +func (db *DB) DeleteFrame(name string) error { + db.mu.Lock() + defer db.mu.Unlock() + + // Ignore if frame doesn't exist. + f := db.frame(name) + if f == nil { + return nil + } + + // Close frame. + if err := f.Close(); err != nil { + return err + } + + // Delete frame directory. + if err := os.RemoveAll(db.FramePath(name)); err != nil { + return err + } + + // Remove reference. + delete(db.frames, name) + + db.stats.Count("frameN", -1) + + return nil +} + +// SetBit sets a bit for a given profile & bitmap. +// If a timestamp is specified then set all bits for the different quantum units. +func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) { + // Read frame. + f, err := db.CreateFrameIfNotExists(name) + if err != nil { + return changed, err + } + + // If this is a non-time bit then simply set the bit on the frame. + if t == nil { + return f.SetBit(bitmapID, profileID) + } + + // Determine quantum of frame. Set to the default quantum if it is unset. + q := f.TimeQuantum() + if q == "" { + q = db.TimeQuantum() + if err := f.SetTimeQuantum(q); err != nil { + return changed, err + } + } + + // If a timestamp is specified then set bits across all frames for the quantum. + for _, subname := range FramesByTime(name, *t, q) { + f, err := db.CreateFrameIfNotExists(subname) + if err != nil { + return changed, err + } + + if c, err := f.SetBit(bitmapID, profileID); err != nil { + return changed, err + } else if c { + changed = true + } + } + return changed, nil +} + type dbSlice []*DB func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/db_test.go b/db_test.go index 5ffc68392..46a138c87 100644 --- a/db_test.go +++ b/db_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "os" "testing" + "time" "github.com/pilosa/pilosa" ) @@ -34,6 +35,49 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) { } } +// Ensure database can delete a frame. +func TestDB_DeleteFrame(t *testing.T) { + db := MustOpenDB() + defer db.Close() + + // Create frame. + if _, err := db.CreateFrameIfNotExists("f"); err != nil { + t.Fatal(err) + } + + // Delete frame & verify it's gone. + if err := db.DeleteFrame("f"); err != nil { + t.Fatal(err) + } else if db.Frame("f") != nil { + t.Fatal("expected nil frame") + } + + // Delete again to make sure it doesn't error. + if err := db.DeleteFrame("f"); err != nil { + t.Fatal(err) + } +} + +// Ensure database can set the default time quantum. +func TestDB_SetTimeQuantum(t *testing.T) { + db := MustOpenDB() + defer db.Close() + + // Set & retrieve time quantum. + if err := db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload database and verify that it is persisted. + if err := db.Reopen(); err != nil { + t.Fatal(err) + } else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} + // DB represents a test wrapper for pilosa.DB. type DB struct { *pilosa.DB @@ -63,3 +107,27 @@ func (db *DB) Close() error { defer os.RemoveAll(db.Path()) return db.DB.Close() } + +// Reopen closes the database and reopens it. +func (db *DB) Reopen() error { + if err := db.DB.Close(); err != nil { + return err + } + + path, name := db.Path(), db.Name() + db.DB = pilosa.NewDB(path, name) + + if err := db.Open(); err != nil { + return err + } + return nil +} + +// MustSetBit sets a bit on the database. Panic on error. +func (db *DB) MustSetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool) { + changed, err := db.SetBit(name, bitmapID, profileID, t) + if err != nil { + panic(err) + } + return changed +} diff --git a/executor.go b/executor.go index abf2d750e..bd0f16aba 100644 --- a/executor.go +++ b/executor.go @@ -304,11 +304,28 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Rang frame = DefaultFrame } - f := e.Index.Fragment(db, frame, slice) + // Retrieve base frame. + f := e.Index.Frame(db, frame) if f == nil { - return NewBitmap(), nil + return &Bitmap{}, nil } - return f.Range(c.ID, c.StartTime, c.EndTime), nil + + // If no quantum exists then return an empty bitmap. + q := f.TimeQuantum() + if q == "" { + return &Bitmap{}, nil + } + + // Union bitmaps across all time-based subframes. + bm := &Bitmap{} + for _, subframe := range FramesByTimeRange(frame, c.StartTime, c.EndTime, q) { + f := e.Index.Fragment(db, subframe, slice) + if f == nil { + continue + } + bm = bm.Union(f.Bitmap(c.ID)) + } + return bm, nil } // executeUnionSlice executes a union() call for a local slice. @@ -405,15 +422,14 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit, for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - f, err := e.Index.CreateFragmentIfNotExists(db, c.Frame, slice) + db, err := e.Index.CreateDBIfNotExists(db) if err != nil { - return false, fmt.Errorf("fragment: %s", err) + return false, fmt.Errorf("db: %s", err) } - val, err := f.SetBit(c.ID, c.ProfileID, opt.Timestamp, opt.Quantum) + val, err := db.SetBit(c.Frame, c.ID, c.ProfileID, opt.Timestamp) if err != nil { return false, err - } - if val { + } else if val { ret = true } continue @@ -579,14 +595,13 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { // Encode request object. pbreq := &internal.QueryRequest{ - DB: proto.String(db), - Query: proto.String(q.String()), - Slices: slices, - Quantum: proto.Uint32(uint32(opt.Quantum)), - Remote: proto.Bool(true), + DB: db, + Query: q.String(), + Slices: slices, + Remote: true, } if opt.Timestamp != nil { - pbreq.Timestamp = proto.Int64(opt.Timestamp.UnixNano()) + pbreq.Timestamp = opt.Timestamp.UnixNano() } buf, err := proto.Marshal(pbreq) if err != nil { @@ -632,7 +647,7 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query } // Return an error, if specified on response. - if err := decodeError(pb.GetErr()); err != nil { + if err := decodeError(pb.Err); err != nil { return nil, err } @@ -648,11 +663,11 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query case *pql.TopN: v, err = decodePairs(pb.Results[i].GetPairs()), nil case *pql.Count: - v, err = pb.Results[i].GetN(), nil + v, err = pb.Results[i].N, nil case *pql.SetBit: - v, err = pb.Results[i].GetChanged(), nil + v, err = pb.Results[i].Changed, nil case *pql.ClearBit: - v, err = pb.Results[i].GetChanged(), nil + v, err = pb.Results[i].Changed, nil case *pql.SetBitmapAttrs: case *pql.SetProfileAttrs: default: @@ -837,7 +852,6 @@ type mapResponse struct { // ExecOptions represents an execution context for a single Execute() call. type ExecOptions struct { Timestamp *time.Time - Quantum TimeQuantum Remote bool } diff --git a/executor_test.go b/executor_test.go index 999c841d0..c0f498dad 100644 --- a/executor_test.go +++ b/executor_test.go @@ -171,15 +171,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -197,12 +197,12 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 2, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth+2, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 2) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -221,19 +221,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2) // Create an intersecting bitmap. - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -253,15 +253,23 @@ func TestExecutor_Execute_Range(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - f := idx.MustCreateFragmentIfNotExists("d", "f.t", 0) - if _, err := f.SetBit(1, 100, MustParseTime("2000-01-01 00:00"), pilosa.YMD); err != nil { - t.Fatal(err) - } + db := idx.MustCreateDBIfNotExists("d") + db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")) + db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-31 00:00")) + db.MustSetBit("f", 1, 3, MustParseTimePtr("2000-01-01 00:00")) + db.MustSetBit("f", 1, 4, MustParseTimePtr("2000-01-02 00:00")) + db.MustSetBit("f", 1, 5, MustParseTimePtr("2000-02-01 00:00")) + db.MustSetBit("f", 1, 6, MustParseTimePtr("2001-01-01 00:00")) + db.MustSetBit("f", 1, 7, MustParseTimePtr("2002-01-01 02:00")) + + db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early + db.MustSetBit("f", 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late + db.MustSetBit("f", 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f.t, start="2000-01-01T00:00", end="2000-01-01T01:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) { + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected bits: %+v", bits) } } diff --git a/fragment.go b/fragment.go index d0bb2e73e..3eea7cf03 100644 --- a/fragment.go +++ b/fragment.go @@ -238,7 +238,7 @@ func (f *Fragment) openCache() error { // Read in all bitmaps by ID. // This will cause them to be added to the cache. - for _, bitmapID := range pb.GetBitmapIDs() { + for _, bitmapID := range pb.BitmapIDs { n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) f.cache.Add(bitmapID, n) } @@ -323,11 +323,6 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { } bm.InvalidateCount() - // f.storage.ForEachRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth, func(i uint64) { - // profileID := (f.slice * SliceWidth) + (i % SliceWidth) - // bm.SetBit(profileID) - // }) - // Update cache. f.cache.Add(bitmapID, bm.Count()) @@ -336,15 +331,9 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { // SetBit sets a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(bitmapID, profileID uint64, t *time.Time, q TimeQuantum) (changed bool, err error) { +func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - - // Set time bits if this is a time-frame and a timestamp is specified. - if strings.HasSuffix(f.frame, FrameSuffixTime) && t != nil { - return f.setTimeBit(bitmapID, profileID, *t, q) - } - return f.setBit(bitmapID, profileID) } @@ -379,17 +368,6 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) return changed, nil } -func (f *Fragment) setTimeBit(bitmapID, profileID uint64, t time.Time, q TimeQuantum) (changed bool, err error) { - for _, timeID := range TimeIDsFromQuantum(q, t, bitmapID) { - if v, err := f.setBit(timeID, profileID); err != nil { - return changed, fmt.Errorf("set time bit: t=%s, q=%s, err=%s", t, q, err) - } else if v { - changed = true - } - } - return changed, nil -} - // ClearBit clears a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) { @@ -601,24 +579,6 @@ type TopOptions struct { FilterValues []interface{} } -func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap { - f.mu.Lock() - defer f.mu.Unlock() - - // Retrieve a list of bitmap ids for a given time range. - bitmapIDs := TimeIDsFromRange(start, end, bitmapID) - if len(bitmapIDs) == 0 { - return NewBitmap() - } - - // Union all bitmap ids from the time range. - bm := f.bitmap(bitmapIDs[0]) - for _, id := range bitmapIDs[1:] { - bm = bm.Union(f.bitmap(id)) - } - return bm -} - // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. func (f *Fragment) Checksum() []byte { diff --git a/fragment_test.go b/fragment_test.go index 11ce52dca..da2edac6c 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -27,11 +27,11 @@ func TestFragment_SetBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(120, 1, nil, 0); err != nil { + if _, err := f.SetBit(120, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 6, nil, 0); err != nil { + } else if _, err := f.SetBit(120, 6); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(121, 0, nil, 0); err != nil { + } else if _, err := f.SetBit(121, 0); err != nil { t.Fatal(err) } @@ -58,9 +58,9 @@ func TestFragment_ClearBit(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1, nil, 0); err != nil { + if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { + } else if _, err := f.SetBit(1000, 2); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -85,9 +85,9 @@ func TestFragment_Snapshot(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1, nil, 0); err != nil { + if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { + } else if _, err := f.SetBit(1000, 2); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -114,11 +114,11 @@ func TestFragment_ForEachBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(100, 20, nil, 0); err != nil { + if _, err := f.SetBit(100, 20); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 38, nil, 0); err != nil { + } else if _, err := f.SetBit(2, 38); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 37, nil, 0); err != nil { + } else if _, err := f.SetBit(2, 37); err != nil { t.Fatal(err) } @@ -284,9 +284,9 @@ func TestFragment_Checksum(t *testing.T) { // Retrieve checksum and set bits. orig := f.Checksum() - if _, err := f.SetBit(1, 200, nil, 0); err != nil { + if _, err := f.SetBit(1, 200); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.HashBlockSize*2, 200, nil, 0); err != nil { + } else if _, err := f.SetBit(pilosa.HashBlockSize*2, 200); err != nil { t.Fatal(err) } @@ -305,7 +305,7 @@ func TestFragment_Blocks(t *testing.T) { var prev []pilosa.FragmentBlock // Set first bit. - if _, err := f.SetBit(0, 0, nil, 0); err != nil { + if _, err := f.SetBit(0, 0); err != nil { t.Fatal(err) } blocks := f.Blocks() @@ -315,7 +315,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different bitmap. - if _, err := f.SetBit(20, 0, nil, 0); err != nil { + if _, err := f.SetBit(20, 0); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -325,7 +325,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different profile. - if _, err := f.SetBit(20, 100, nil, 0); err != nil { + if _, err := f.SetBit(20, 100); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -340,7 +340,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { defer f.Close() // Set bits on a different block. - if _, err := f.SetBit(100, 1, nil, 0); err != nil { + if _, err := f.SetBit(100, 1); err != nil { t.Fatal(err) } @@ -359,7 +359,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0, nil, 0); err != nil { + if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) } } @@ -391,7 +391,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0, nil, 0); err != nil { + if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) } } @@ -422,9 +422,9 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { defer f0.Close() // Set and then clear bits on the fragment. - if _, err := f0.SetBit(1000, 1, nil, 0); err != nil { + if _, err := f0.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f0.SetBit(1000, 2, nil, 0); err != nil { + } else if _, err := f0.SetBit(1000, 2); err != nil { t.Fatal(err) } else if _, err := f0.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -527,12 +527,12 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Generate some intersecting data. for i := 0; i < 10000; i += 2 { - if _, err := f.SetBit(1, uint64(i), nil, 0); err != nil { + if _, err := f.SetBit(1, uint64(i)); err != nil { b.Fatal(err) } } for i := 0; i < 10000; i += 3 { - if _, err := f.SetBit(2, uint64(i), nil, 0); err != nil { + if _, err := f.SetBit(2, uint64(i)); err != nil { b.Fatal(err) } } @@ -609,7 +609,7 @@ func (f *Fragment) Reopen() error { // This function does not accept a timestamp or quantum. func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { for _, profileID := range profileIDs { - if _, err := f.SetBit(bitmapID, profileID, nil, 0); err != nil { + if _, err := f.SetBit(bitmapID, profileID); err != nil { panic(err) } } diff --git a/frame.go b/frame.go index 56e4b3f1a..a3e90ce1e 100644 --- a/frame.go +++ b/frame.go @@ -2,26 +2,28 @@ package pilosa import ( "fmt" + "io/ioutil" "os" "path/filepath" "strconv" "sync" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) const ( - // FrameSuffixTime is the suffix used for time-based frames. - FrameSuffixTime = ".t" - // FrameSuffixRank is the suffix used for rank-based frames. FrameSuffixRank = ".n" ) // Frame represents a container for fragments. type Frame struct { - mu sync.Mutex - path string - db string - name string + mu sync.Mutex + path string + db string + name string + timeQuantum TimeQuantum // Fragments by slice. fragments map[uint64]*Fragment @@ -80,6 +82,10 @@ func (f *Frame) Open() error { return err } + if err := f.loadMeta(); err != nil { + return err + } + if err := f.openFragments(); err != nil { return err } @@ -134,6 +140,45 @@ func (f *Frame) openFragments() error { return nil } +// loadMeta reads meta data for the frame, if any. +func (f *Frame) loadMeta() error { + var pb internal.Frame + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(f.path, "meta")) + if os.IsNotExist(err) { + f.timeQuantum = "" + return nil + } else if err != nil { + return err + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + } + + // Copy metadata fields. + f.timeQuantum = TimeQuantum(pb.TimeQuantum) + + return nil +} + +// saveMeta writes meta data for the frame. +func (f *Frame) saveMeta() error { + // Marshal metadata. + buf, err := proto.Marshal(&internal.Frame{TimeQuantum: string(f.timeQuantum)}) + if err != nil { + return err + } + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(f.path, "meta"), buf, 0666); err != nil { + return err + } + + return nil +} + // Close closes the frame and its fragments. func (f *Frame) Close() error { f.mu.Lock() @@ -153,6 +198,34 @@ func (f *Frame) Close() error { return nil } +// TimeQuantum returns the time quantum for the frame. +func (f *Frame) TimeQuantum() TimeQuantum { + f.mu.Lock() + defer f.mu.Unlock() + return f.timeQuantum +} + +// SetTimeQuantum sets the time quantum for the frame. +func (f *Frame) SetTimeQuantum(q TimeQuantum) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Validate input. + if !q.Valid() { + return ErrInvalidTimeQuantum + } + + // Update value on frame. + f.timeQuantum = q + + // Perist meta data to disk. + if err := f.saveMeta(); err != nil { + return err + } + + return nil +} + // FragmentPath returns the path to a fragment in the frame. func (f *Frame) FragmentPath(slice uint64) string { return filepath.Join(f.path, strconv.FormatUint(slice, 10)) @@ -213,6 +286,16 @@ func (f *Frame) newFragment(path string, slice uint64) *Fragment { return frag } +// SetBit sets a bit within the frame. +func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) { + slice := bitmapID / SliceWidth + frag, err := f.CreateFragmentIfNotExists(slice) + if err != nil { + return changed, err + } + return frag.SetBit(bitmapID, profileID) +} + type frameSlice []*Frame func (p frameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/frame_test.go b/frame_test.go index 01d508ef9..186db0c62 100644 --- a/frame_test.go +++ b/frame_test.go @@ -34,6 +34,26 @@ func TestFrame_CreateFragmentIfNotExists(t *testing.T) { } } +// Ensure frame can set its time quantum. +func TestFrame_SetTimeQuantum(t *testing.T) { + f := MustOpenFrame() + defer f.Close() + + // Set & retrieve time quantum. + if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload frame and verify that it is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} + // Frame represents a test wrapper for pilosa.Frame. type Frame struct { *pilosa.Frame @@ -63,3 +83,18 @@ func (f *Frame) Close() error { defer os.RemoveAll(f.Path()) return f.Frame.Close() } + +// Reopen closes the database and reopens it. +func (f *Frame) Reopen() error { + if err := f.Frame.Close(); err != nil { + return err + } + + path, db, name := f.Path(), f.DB(), f.Name() + f.Frame = pilosa.NewFrame(path, db, name) + + if err := f.Open(); err != nil { + return err + } + return nil +} diff --git a/handler.go b/handler.go index 9899133db..f9088deb1 100644 --- a/handler.go +++ b/handler.go @@ -114,6 +114,13 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } + case "/db/time_quantum": + switch r.Method { + case "PATCH": + h.handlePatchDBTimeQuantum(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } case "/db/attr/diff": switch r.Method { case "POST": @@ -121,6 +128,20 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } + case "/frame": + switch r.Method { + case "DELETE": + h.handleDeleteFrame(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } + case "/frame/time_quantum": + switch r.Method { + case "PATCH": + h.handlePatchFrameTimeQuantum(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } case "/frame/attr/diff": switch r.Method { case "POST": @@ -203,7 +224,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // Build execution options. opt := &ExecOptions{ Timestamp: req.Timestamp, - Quantum: req.Quantum, Remote: req.Remote, } @@ -256,7 +276,7 @@ func (h *Handler) handleGetSliceMax(w http.ResponseWriter, r *http.Request) erro sm := h.Index.SliceN() if strings.Contains(r.Header.Get("Accept"), "application/x-protobuf") { pb := &internal.SliceMaxResponse{ - SliceMax: &sm, + SliceMax: sm, } if buf, err := proto.Marshal(pb); err != nil { return err @@ -299,6 +319,48 @@ type deleteDBRequest struct { type deleteDBResponse struct{} +// handlePatchDBTimeQuantum handles PATCH /db/time_quantum request. +func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req patchDBTimeQuantumRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Validate quantum. + tq, err := ParseTimeQuantum(req.TimeQuantum) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Retrieve database by name. + db, err := h.Index.CreateDBIfNotExists(req.DB) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Set default time quantum on database. + if err := db.SetTimeQuantum(tq); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(patchDBTimeQuantumResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type patchDBTimeQuantumRequest struct { + DB string `json:"db"` + TimeQuantum string `json:"time_quantum"` +} + +type patchDBTimeQuantumResponse struct{} + // handlePostDBAttrDiff handles POST /db/attr/diff requests. func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -355,6 +417,86 @@ type postDBAttrDiffResponse struct { Attrs map[uint64]map[string]interface{} `json:"attrs"` } +// handleDeleteFrame handles DELETE /frame request. +func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req deleteFrameRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Find database. + db := h.Index.DB(req.DB) + if db == nil { + if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } + return + } + + // Delete frame from the database. + if err := db.DeleteFrame(req.Frame); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type deleteFrameRequest struct { + DB string `json:"db"` + Frame string `json:"frame"` +} + +type deleteFrameResponse struct{} + +// handlePatchFrameTimeQuantum handles PATCH /frame/time_quantum request. +func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req patchFrameTimeQuantumRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Validate quantum. + tq, err := ParseTimeQuantum(req.TimeQuantum) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Retrieve database by name. + f, err := h.Index.CreateFrameIfNotExists(req.DB, req.Frame) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Set default time quantum on database. + if err := f.SetTimeQuantum(tq); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type patchFrameTimeQuantumRequest struct { + DB string `json:"db"` + Frame string `json:"frame"` + TimeQuantum string `json:"time_quantum"` +} + +type patchFrameTimeQuantumResponse struct{} + // handlePostFrameAttrDiff handles POST /frame/attr/diff requests. func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -495,7 +637,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { } // Parse time granularity. - quantum := YMDH + quantum := TimeQuantum("YMDH") if s := q.Get("time_granularity"); s != "" { v, err := ParseTimeQuantum(s) if err != nil { @@ -561,7 +703,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - db, frame, slice := req.GetDB(), req.GetFrame(), req.GetSlice() + db, frame, slice := req.DB, req.Frame, req.Slice // Validate that this handler owns the slice. if !h.Cluster.OwnsFragment(h.Host, db, slice) { @@ -578,16 +720,16 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { http.Error(w, "fragment error", http.StatusInternalServerError) return } - h.logger().Println("Import into Fragment:", db, frame, slice, len(req.GetProfileIDs())) + h.logger().Println("Import into Fragment:", db, frame, slice, len(req.ProfileIDs)) // Import into fragment. - err = f.Import(req.GetBitmapIDs(), req.GetProfileIDs()) + err = f.Import(req.BitmapIDs, req.ProfileIDs) if err != nil { - h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", db, frame, slice, len(req.GetProfileIDs()), err) + h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", db, frame, slice, len(req.ProfileIDs), err) } // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: proto.String(errorString(err))}) + buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) if e != nil { http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) return @@ -733,7 +875,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ } // Retrieve fragment from index. - f := h.Index.Fragment(req.GetDB(), req.GetFrame(), req.GetSlice()) + f := h.Index.Fragment(req.DB, req.Frame, req.Slice) if f == nil { http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) return @@ -742,7 +884,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ // Read data var resp internal.BlockDataResponse if f != nil { - resp.BitmapIDs, resp.ProfileIDs = f.BlockData(int(req.GetBlock())) + resp.BitmapIDs, resp.ProfileIDs = f.BlockData(int(req.Block)) } // Encode response. @@ -919,16 +1061,16 @@ type QueryRequest struct { func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { req := &QueryRequest{ - DB: pb.GetDB(), - Query: pb.GetQuery(), - Slices: pb.GetSlices(), - Profiles: pb.GetProfiles(), - Quantum: TimeQuantum(pb.GetQuantum()), - Remote: pb.GetRemote(), + DB: pb.DB, + Query: pb.Query, + Slices: pb.Slices, + Profiles: pb.Profiles, + Quantum: TimeQuantum(pb.Quantum), + Remote: pb.Remote, } - if pb.Timestamp != nil { - t := time.Unix(0, pb.GetTimestamp()) + if pb.Timestamp != 0 { + t := time.Unix(0, pb.Timestamp) req.Timestamp = &t } @@ -978,14 +1120,14 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { case []Pair: pb.Results[i].Pairs = encodePairs(result) case uint64: - pb.Results[i].N = proto.Uint64(result) + pb.Results[i].N = result case bool: - pb.Results[i].Changed = proto.Bool(result) + pb.Results[i].Changed = result } } if resp.Err != nil { - pb.Err = proto.String(resp.Err.Error()) + pb.Err = resp.Err.Error() } return pb diff --git a/handler_test.go b/handler_test.go index 90ecfa608..237133d78 100644 --- a/handler_test.go +++ b/handler_test.go @@ -93,8 +93,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Generate request body. reqBody, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String("db0"), - Query: proto.String("Count(Bitmap(100))"), + DB: "db0", + Query: "Count(Bitmap(100))", Slices: []uint64{0, 1}, }) if err != nil { @@ -157,7 +157,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if n := resp.Results[0].GetN(); n != 100 { + } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) } } @@ -232,15 +232,15 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { + } else if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) - } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { + } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].GetKey(), attrs[1].GetUintValue(); k != "c" || v != uint64(1) { + } else if k, v := attrs[1].Key, attrs[1].UintValue; k != "c" || v != uint64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].GetKey(), attrs[2].GetBoolValue(); k != "d" || v != true { + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v) } } @@ -268,9 +268,9 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Encode request body. buf, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String("d"), - Query: proto.String("Bitmap(100)"), - Profiles: proto.Bool(true), + DB: "d", + Query: "Bitmap(100)", + Profiles: true, }) if err != nil { t.Fatal(err) @@ -289,25 +289,25 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { + if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) - } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { + } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].GetKey(), attrs[1].GetUintValue(); k != "c" || v != uint64(1) { + } else if k, v := attrs[1].Key, attrs[1].UintValue; k != "c" || v != uint64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].GetKey(), attrs[2].GetBoolValue(); k != "d" || v != true { + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v) } - if a := resp.GetProfiles(); len(a) != 1 { + if a := resp.Profiles; len(a) != 1 { t.Fatalf("unexpected profiles length: %d", len(a)) - } else if a[0].GetID() != 1 { - t.Fatalf("unexpected id: %d", a[0].GetID()) - } else if len(a[0].GetAttrs()) != 1 { + } else if a[0].ID != 1 { + t.Fatalf("unexpected id: %d", a[0].ID) + } else if len(a[0].Attrs) != 1 { t.Fatalf("unexpected profile attr length: %d", len(a)) - } else if k, v := a[0].GetAttrs()[0].GetKey(), a[0].GetAttrs()[0].GetStringValue(); k != "x" || v != "y" { + } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) } } @@ -391,7 +391,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if s := resp.GetErr(); s != `marker` { + } else if s := resp.Err; s != `marker` { t.Fatalf("unexpected error: %s", s) } } @@ -453,6 +453,63 @@ func TestHandler_DB_Delete(t *testing.T) { } } +// Ensure handler can delete a frame. +func TestHandler_DeleteFrame(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + if _, err := idx.CreateFrameIfNotExists("d0", "f1"); err != nil { + t.Fatal(err) + } + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/frame", strings.NewReader(`{"db":"d0","frame":"f1"}`))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if f := idx.DB("d0").Frame("f1"); f != nil { + t.Fatal("expected nil frame") + } +} + +// Ensure handler can set the DB time quantum. +func TestHandler_SetDBTimeQuantum(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/time_quantum", strings.NewReader(`{"db":"d0","time_quantum":"ymdh"}`))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if q := idx.DB("d0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected time quantum: %s", q) + } +} + +// Ensure handler can set the frame time quantum. +func TestHandler_SetFrameTimeQuantum(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/frame/time_quantum", strings.NewReader(`{"db":"d0","frame":"f1","time_quantum":"ymdh"}`))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if q := idx.DB("d0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected time quantum: %s", q) + } +} + // Ensure the handler can return data in differing blocks for a database. func TestHandler_DB_AttrStore_Diff(t *testing.T) { idx := MustOpenIndex() diff --git a/index_test.go b/index_test.go index 851de772f..512c211c6 100644 --- a/index_test.go +++ b/index_test.go @@ -19,11 +19,11 @@ func TestIndex_DeleteDB(t *testing.T) { // Write bits to separate databases. f0 := idx.MustCreateFragmentIfNotExists("d0", "f", 0) - if _, err := f0.SetBit(100, 200, nil, 0); err != nil { + if _, err := f0.SetBit(100, 200); err != nil { t.Fatal(err) } f1 := idx.MustCreateFragmentIfNotExists("d1", "f", 0) - if _, err := f1.SetBit(100, 200, nil, 0); err != nil { + if _, err := f1.SetBit(100, 200); err != nil { t.Fatal(err) } @@ -74,18 +74,18 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Set data on the local index. f := idx0.MustCreateFragmentIfNotExists("d", "f", 0) - if _, err := f.SetBit(0, 10, nil, 0); err != nil { + if _, err := f.SetBit(0, 10); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 20, nil, 0); err != nil { + } else if _, err := f.SetBit(2, 20); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 10, nil, 0); err != nil { + } else if _, err := f.SetBit(120, 10); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(200, 4, nil, 0); err != nil { + } else if _, err := f.SetBit(200, 4); err != nil { t.Fatal(err) } f = idx0.MustCreateFragmentIfNotExists("d", "f0", 1) - if _, err := f.SetBit(9, SliceWidth+5, nil, 0); err != nil { + if _, err := f.SetBit(9, SliceWidth+5); err != nil { t.Fatal(err) } @@ -93,20 +93,20 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Set data on the remote index. f = idx1.MustCreateFragmentIfNotExists("d", "f", 0) - if _, err := f.SetBit(0, 4000, nil, 0); err != nil { + if _, err := f.SetBit(0, 4000); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(3, 10, nil, 0); err != nil { + } else if _, err := f.SetBit(3, 10); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 10, nil, 0); err != nil { + } else if _, err := f.SetBit(120, 10); err != nil { t.Fatal(err) } f = idx1.MustCreateFragmentIfNotExists("y", "z", 3) - if _, err := f.SetBit(10, (3*SliceWidth)+4, nil, 0); err != nil { + if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+5, nil, 0); err != nil { + } else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+7, nil, 0); err != nil { + } else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil { t.Fatal(err) } @@ -185,6 +185,24 @@ func (i *Index) Close() error { return i.Index.Close() } +// MustCreateDBIfNotExists returns a given db. Panic on error. +func (i *Index) MustCreateDBIfNotExists(db string) *DB { + d, err := i.Index.CreateDBIfNotExists(db) + if err != nil { + panic(err) + } + return &DB{DB: d} +} + +// MustCreateFrameIfNotExists returns a given frame. Panic on error. +func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame { + f, err := i.Index.CreateFrameIfNotExists(db, frame) + if err != nil { + panic(err) + } + return &Frame{Frame: f} +} + // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. func (i *Index) MustCreateFragmentIfNotExists(db, frame string, slice uint64) *Fragment { f, err := i.Index.CreateFragmentIfNotExists(db, frame, slice) diff --git a/internal/internal.go b/internal/internal.go index 574c0ee5c..3d322d82a 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -7,6 +7,8 @@ import ( "github.com/gogo/protobuf/proto" ) +//go:generate protoc --gofast_out=. internal.proto + type Request proto.Message type Response proto.Message diff --git a/internal/internal.pb.go b/internal/internal.pb.go index c61ad59c7..c177fdd24 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -1,36 +1,40 @@ // Code generated by protoc-gen-gogo. -// source: internal/internal.proto +// source: internal.proto // DO NOT EDIT! /* -Package internal is a generated protocol buffer package. + Package internal is a generated protocol buffer package. -It is generated from these files: - internal/internal.proto + It is generated from these files: + internal.proto -It has these top-level messages: - Bitmap - Pair - Bit - Profile - Attr - AttrMap - QueryRequest - QueryResponse - QueryResult - ImportRequest - ImportResponse - BlockDataRequest - BlockDataResponse - Cache - SliceMaxResponse + It has these top-level messages: + DB + Frame + Bitmap + Pair + Bit + Profile + Attr + AttrMap + QueryRequest + QueryResponse + QueryResult + ImportRequest + ImportResponse + BlockDataRequest + BlockDataResponse + Cache + SliceMaxResponse */ package internal -import proto "github.com/gogo/protobuf/proto" +import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import io "io" + // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf @@ -38,25 +42,37 @@ var _ = math.Inf // This is a compile-time assertion to ensure that this generated file // is compatible with the proto package it is being compiled against. -const _ = proto.GoGoProtoPackageIsVersion1 +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type DB struct { + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` +} + +func (m *DB) Reset() { *m = DB{} } +func (m *DB) String() string { return proto.CompactTextString(m) } +func (*DB) ProtoMessage() {} +func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } + +type Frame struct { + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` +} + +func (m *Frame) Reset() { *m = Frame{} } +func (m *Frame) String() string { return proto.CompactTextString(m) } +func (*Frame) ProtoMessage() {} +func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } type Bitmap struct { - Bits []uint64 `protobuf:"varint,1,rep,name=Bits" json:"Bits,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_unrecognized []byte `json:"-"` + Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Bitmap) Reset() { *m = Bitmap{} } func (m *Bitmap) String() string { return proto.CompactTextString(m) } func (*Bitmap) ProtoMessage() {} -func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } - -func (m *Bitmap) GetBits() []uint64 { - if m != nil { - return m.Bits - } - return nil -} +func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} } func (m *Bitmap) GetAttrs() []*Attr { if m != nil { @@ -66,72 +82,34 @@ func (m *Bitmap) GetAttrs() []*Attr { } type Pair struct { - Key *uint64 `protobuf:"varint,1,req,name=Key" json:"Key,omitempty"` - Count *uint64 `protobuf:"varint,2,req,name=Count" json:"Count,omitempty"` - XXX_unrecognized []byte `json:"-"` + Key uint64 `protobuf:"varint,1,opt,name=Key,proto3" json:"Key,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} -func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } - -func (m *Pair) GetKey() uint64 { - if m != nil && m.Key != nil { - return *m.Key - } - return 0 -} - -func (m *Pair) GetCount() uint64 { - if m != nil && m.Count != nil { - return *m.Count - } - return 0 -} +func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } type Bit struct { - BitmapID *uint64 `protobuf:"varint,1,req,name=BitmapID" json:"BitmapID,omitempty"` - ProfileID *uint64 `protobuf:"varint,2,req,name=ProfileID" json:"ProfileID,omitempty"` - XXX_unrecognized []byte `json:"-"` + BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,proto3" json:"BitmapID,omitempty"` + ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,proto3" json:"ProfileID,omitempty"` } func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} } - -func (m *Bit) GetBitmapID() uint64 { - if m != nil && m.BitmapID != nil { - return *m.BitmapID - } - return 0 -} - -func (m *Bit) GetProfileID() uint64 { - if m != nil && m.ProfileID != nil { - return *m.ProfileID - } - return 0 -} +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } type Profile struct { - ID *uint64 `protobuf:"varint,1,req,name=ID" json:"ID,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_unrecognized []byte `json:"-"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Profile) Reset() { *m = Profile{} } func (m *Profile) String() string { return proto.CompactTextString(m) } func (*Profile) ProtoMessage() {} -func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } - -func (m *Profile) GetID() uint64 { - if m != nil && m.ID != nil { - return *m.ID - } - return 0 -} +func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} } func (m *Profile) GetAttrs() []*Attr { if m != nil { @@ -141,55 +119,26 @@ func (m *Profile) GetAttrs() []*Attr { } type Attr struct { - Key *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"` - StringValue *string `protobuf:"bytes,2,opt,name=StringValue" json:"StringValue,omitempty"` - UintValue *uint64 `protobuf:"varint,3,opt,name=UintValue" json:"UintValue,omitempty"` - BoolValue *bool `protobuf:"varint,4,opt,name=BoolValue" json:"BoolValue,omitempty"` - XXX_unrecognized []byte `json:"-"` + Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` + Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` + StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` + UintValue uint64 `protobuf:"varint,4,opt,name=UintValue,proto3" json:"UintValue,omitempty"` + BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` } func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } - -func (m *Attr) GetKey() string { - if m != nil && m.Key != nil { - return *m.Key - } - return "" -} - -func (m *Attr) GetStringValue() string { - if m != nil && m.StringValue != nil { - return *m.StringValue - } - return "" -} - -func (m *Attr) GetUintValue() uint64 { - if m != nil && m.UintValue != nil { - return *m.UintValue - } - return 0 -} - -func (m *Attr) GetBoolValue() bool { - if m != nil && m.BoolValue != nil { - return *m.BoolValue - } - return false -} +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` - XXX_unrecognized []byte `json:"-"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -199,88 +148,30 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Query *string `protobuf:"bytes,2,req,name=Query" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,3,rep,name=Slices" json:"Slices,omitempty"` - Profiles *bool `protobuf:"varint,4,opt,name=Profiles" json:"Profiles,omitempty"` - Timestamp *int64 `protobuf:"varint,5,opt,name=Timestamp" json:"Timestamp,omitempty"` - Quantum *uint32 `protobuf:"varint,6,opt,name=Quantum" json:"Quantum,omitempty"` - Remote *bool `protobuf:"varint,7,opt,name=Remote" json:"Remote,omitempty"` - XXX_unrecognized []byte `json:"-"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Query string `protobuf:"bytes,2,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,3,rep,packed,name=Slices" json:"Slices,omitempty"` + Profiles bool `protobuf:"varint,4,opt,name=Profiles,proto3" json:"Profiles,omitempty"` + Timestamp int64 `protobuf:"varint,5,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` + Quantum string `protobuf:"bytes,6,opt,name=Quantum,proto3" json:"Quantum,omitempty"` + Remote bool `protobuf:"varint,7,opt,name=Remote,proto3" json:"Remote,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } - -func (m *QueryRequest) GetDB() string { - if m != nil && m.DB != nil { - return *m.DB - } - return "" -} - -func (m *QueryRequest) GetQuery() string { - if m != nil && m.Query != nil { - return *m.Query - } - return "" -} - -func (m *QueryRequest) GetSlices() []uint64 { - if m != nil { - return m.Slices - } - return nil -} - -func (m *QueryRequest) GetProfiles() bool { - if m != nil && m.Profiles != nil { - return *m.Profiles - } - return false -} - -func (m *QueryRequest) GetTimestamp() int64 { - if m != nil && m.Timestamp != nil { - return *m.Timestamp - } - return 0 -} - -func (m *QueryRequest) GetQuantum() uint32 { - if m != nil && m.Quantum != nil { - return *m.Quantum - } - return 0 -} - -func (m *QueryRequest) GetRemote() bool { - if m != nil && m.Remote != nil { - return *m.Remote - } - return false -} +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } type QueryResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` - Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` - XXX_unrecognized []byte `json:"-"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` } func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} } - -func (m *QueryResponse) GetErr() string { - if m != nil && m.Err != nil { - return *m.Err - } - return "" -} +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} } func (m *QueryResponse) GetResults() []*QueryResult { if m != nil { @@ -297,17 +188,16 @@ func (m *QueryResponse) GetProfiles() []*Profile { } type QueryResult struct { - Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` - N *uint64 `protobuf:"varint,2,opt,name=N" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` - Changed *bool `protobuf:"varint,4,opt,name=Changed" json:"Changed,omitempty"` - XXX_unrecognized []byte `json:"-"` + Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} } func (m *QueryResult) GetBitmap() *Bitmap { if m != nil { @@ -316,13 +206,6 @@ func (m *QueryResult) GetBitmap() *Bitmap { return nil } -func (m *QueryResult) GetN() uint64 { - if m != nil && m.N != nil { - return *m.N - } - return 0 -} - func (m *QueryResult) GetPairs() []*Pair { if m != nil { return m.Pairs @@ -330,180 +213,71 @@ func (m *QueryResult) GetPairs() []*Pair { return nil } -func (m *QueryResult) GetChanged() bool { - if m != nil && m.Changed != nil { - return *m.Changed - } - return false -} - type ImportRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Frame *string `protobuf:"bytes,2,req,name=Frame" json:"Frame,omitempty"` - Slice *uint64 `protobuf:"varint,3,req,name=Slice" json:"Slice,omitempty"` - BitmapIDs []uint64 `protobuf:"varint,4,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,5,rep,name=ProfileIDs" json:"ProfileIDs,omitempty"` - XXX_unrecognized []byte `json:"-"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` } func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} } - -func (m *ImportRequest) GetDB() string { - if m != nil && m.DB != nil { - return *m.DB - } - return "" -} - -func (m *ImportRequest) GetFrame() string { - if m != nil && m.Frame != nil { - return *m.Frame - } - return "" -} - -func (m *ImportRequest) GetSlice() uint64 { - if m != nil && m.Slice != nil { - return *m.Slice - } - return 0 -} - -func (m *ImportRequest) GetBitmapIDs() []uint64 { - if m != nil { - return m.BitmapIDs - } - return nil -} - -func (m *ImportRequest) GetProfileIDs() []uint64 { - if m != nil { - return m.ProfileIDs - } - return nil -} +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } type ImportResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err" json:"Err,omitempty"` - XXX_unrecognized []byte `json:"-"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` } func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} -func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} } - -func (m *ImportResponse) GetErr() string { - if m != nil && m.Err != nil { - return *m.Err - } - return "" -} +func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } type BlockDataRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB" json:"DB,omitempty"` - Frame *string `protobuf:"bytes,2,req,name=Frame" json:"Frame,omitempty"` - Slice *uint64 `protobuf:"varint,3,req,name=Slice" json:"Slice,omitempty"` - Block *uint64 `protobuf:"varint,4,req,name=Block" json:"Block,omitempty"` - XXX_unrecognized []byte `json:"-"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` } func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} -func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } - -func (m *BlockDataRequest) GetDB() string { - if m != nil && m.DB != nil { - return *m.DB - } - return "" -} - -func (m *BlockDataRequest) GetFrame() string { - if m != nil && m.Frame != nil { - return *m.Frame - } - return "" -} - -func (m *BlockDataRequest) GetSlice() uint64 { - if m != nil && m.Slice != nil { - return *m.Slice - } - return 0 -} - -func (m *BlockDataRequest) GetBlock() uint64 { - if m != nil && m.Block != nil { - return *m.Block - } - return 0 -} +func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } type BlockDataResponse struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,2,rep,name=ProfileIDs" json:"ProfileIDs,omitempty"` - XXX_unrecognized []byte `json:"-"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` } func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} -func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } - -func (m *BlockDataResponse) GetBitmapIDs() []uint64 { - if m != nil { - return m.BitmapIDs - } - return nil -} - -func (m *BlockDataResponse) GetProfileIDs() []uint64 { - if m != nil { - return m.ProfileIDs - } - return nil -} +func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } type Cache struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs" json:"BitmapIDs,omitempty"` - XXX_unrecognized []byte `json:"-"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` } func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} -func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } - -func (m *Cache) GetBitmapIDs() []uint64 { - if m != nil { - return m.BitmapIDs - } - return nil -} +func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{15} } type SliceMaxResponse struct { - SliceMax *uint64 `protobuf:"varint,1,req,name=SliceMax" json:"SliceMax,omitempty"` - XXX_unrecognized []byte `json:"-"` + SliceMax uint64 `protobuf:"varint,1,opt,name=SliceMax,proto3" json:"SliceMax,omitempty"` } func (m *SliceMaxResponse) Reset() { *m = SliceMaxResponse{} } func (m *SliceMaxResponse) String() string { return proto.CompactTextString(m) } func (*SliceMaxResponse) ProtoMessage() {} -func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } - -func (m *SliceMaxResponse) GetSliceMax() uint64 { - if m != nil && m.SliceMax != nil { - return *m.SliceMax - } - return 0 -} +func (*SliceMaxResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{16} } func init() { + proto.RegisterType((*DB)(nil), "internal.DB") + proto.RegisterType((*Frame)(nil), "internal.Frame") proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*Bit)(nil), "internal.Bit") @@ -520,39 +294,3384 @@ func init() { proto.RegisterType((*Cache)(nil), "internal.Cache") proto.RegisterType((*SliceMaxResponse)(nil), "internal.SliceMaxResponse") } +func (m *DB) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *DB) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } + return i, nil +} + +func (m *Frame) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Frame) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } + return i, nil +} + +func (m *Bitmap) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Bitmap) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Bits) > 0 { + dAtA2 := make([]byte, len(m.Bits)*10) + var j1 int + for _, num := range m.Bits { + for num >= 1<<7 { + dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j1++ + } + dAtA2[j1] = uint8(num) + j1++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(j1)) + i += copy(dAtA[i:], dAtA2[:j1]) + } + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Pair) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Pair) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Key != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Key)) + } + if m.Count != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Count)) + } + return i, nil +} + +func (m *Bit) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Bit) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.BitmapID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.BitmapID)) + } + if m.ProfileID != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.ProfileID)) + } + return i, nil +} + +func (m *Profile) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Profile) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.ID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.ID)) + } + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Attr) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Attr) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Key) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } + if m.Type != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Type)) + } + if len(m.StringValue) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.StringValue))) + i += copy(dAtA[i:], m.StringValue) + } + if m.UintValue != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.UintValue)) + } + if m.BoolValue { + dAtA[i] = 0x28 + i++ + if m.BoolValue { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *AttrMap) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AttrMap) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *QueryRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Query) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Query))) + i += copy(dAtA[i:], m.Query) + } + if len(m.Slices) > 0 { + dAtA4 := make([]byte, len(m.Slices)*10) + var j3 int + for _, num := range m.Slices { + for num >= 1<<7 { + dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j3++ + } + dAtA4[j3] = uint8(num) + j3++ + } + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(j3)) + i += copy(dAtA[i:], dAtA4[:j3]) + } + if m.Profiles { + dAtA[i] = 0x20 + i++ + if m.Profiles { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.Timestamp != 0 { + dAtA[i] = 0x28 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Timestamp)) + } + if len(m.Quantum) > 0 { + dAtA[i] = 0x32 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Quantum))) + i += copy(dAtA[i:], m.Quantum) + } + if m.Remote { + dAtA[i] = 0x38 + i++ + if m.Remote { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *QueryResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Err) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Err))) + i += copy(dAtA[i:], m.Err) + } + if len(m.Results) > 0 { + for _, msg := range m.Results { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.Profiles) > 0 { + for _, msg := range m.Profiles { + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *QueryResult) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Bitmap != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Bitmap.Size())) + n5, err := m.Bitmap.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n5 + } + if m.N != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.N)) + } + if len(m.Pairs) > 0 { + for _, msg := range m.Pairs { + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Changed { + dAtA[i] = 0x20 + i++ + if m.Changed { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *ImportRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Slice != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Slice)) + } + if len(m.BitmapIDs) > 0 { + dAtA7 := make([]byte, len(m.BitmapIDs)*10) + var j6 int + for _, num := range m.BitmapIDs { + for num >= 1<<7 { + dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j6++ + } + dAtA7[j6] = uint8(num) + j6++ + } + dAtA[i] = 0x22 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j6)) + i += copy(dAtA[i:], dAtA7[:j6]) + } + if len(m.ProfileIDs) > 0 { + dAtA9 := make([]byte, len(m.ProfileIDs)*10) + var j8 int + for _, num := range m.ProfileIDs { + for num >= 1<<7 { + dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j8++ + } + dAtA9[j8] = uint8(num) + j8++ + } + dAtA[i] = 0x2a + i++ + i = encodeVarintInternal(dAtA, i, uint64(j8)) + i += copy(dAtA[i:], dAtA9[:j8]) + } + return i, nil +} + +func (m *ImportResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Err) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Err))) + i += copy(dAtA[i:], m.Err) + } + return i, nil +} + +func (m *BlockDataRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Slice != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Slice)) + } + if m.Block != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Block)) + } + return i, nil +} + +func (m *BlockDataResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.BitmapIDs) > 0 { + dAtA11 := make([]byte, len(m.BitmapIDs)*10) + var j10 int + for _, num := range m.BitmapIDs { + for num >= 1<<7 { + dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j10++ + } + dAtA11[j10] = uint8(num) + j10++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(j10)) + i += copy(dAtA[i:], dAtA11[:j10]) + } + if len(m.ProfileIDs) > 0 { + dAtA13 := make([]byte, len(m.ProfileIDs)*10) + var j12 int + for _, num := range m.ProfileIDs { + for num >= 1<<7 { + dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j12++ + } + dAtA13[j12] = uint8(num) + j12++ + } + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j12)) + i += copy(dAtA[i:], dAtA13[:j12]) + } + return i, nil +} + +func (m *Cache) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Cache) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.BitmapIDs) > 0 { + dAtA15 := make([]byte, len(m.BitmapIDs)*10) + var j14 int + for _, num := range m.BitmapIDs { + for num >= 1<<7 { + dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j14++ + } + dAtA15[j14] = uint8(num) + j14++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(j14)) + i += copy(dAtA[i:], dAtA15[:j14]) + } + return i, nil +} + +func (m *SliceMaxResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *SliceMaxResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.SliceMax != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.SliceMax)) + } + return i, nil +} + +func encodeFixed64Internal(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Internal(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintInternal(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *DB) Size() (n int) { + var l int + _ = l + l = len(m.TimeQuantum) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + return n +} + +func (m *Frame) Size() (n int) { + var l int + _ = l + l = len(m.TimeQuantum) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + return n +} + +func (m *Bitmap) Size() (n int) { + var l int + _ = l + if len(m.Bits) > 0 { + l = 0 + for _, e := range m.Bits { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if len(m.Attrs) > 0 { + for _, e := range m.Attrs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *Pair) Size() (n int) { + var l int + _ = l + if m.Key != 0 { + n += 1 + sovInternal(uint64(m.Key)) + } + if m.Count != 0 { + n += 1 + sovInternal(uint64(m.Count)) + } + return n +} + +func (m *Bit) Size() (n int) { + var l int + _ = l + if m.BitmapID != 0 { + n += 1 + sovInternal(uint64(m.BitmapID)) + } + if m.ProfileID != 0 { + n += 1 + sovInternal(uint64(m.ProfileID)) + } + return n +} + +func (m *Profile) Size() (n int) { + var l int + _ = l + if m.ID != 0 { + n += 1 + sovInternal(uint64(m.ID)) + } + if len(m.Attrs) > 0 { + for _, e := range m.Attrs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *Attr) Size() (n int) { + var l int + _ = l + l = len(m.Key) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Type != 0 { + n += 1 + sovInternal(uint64(m.Type)) + } + l = len(m.StringValue) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.UintValue != 0 { + n += 1 + sovInternal(uint64(m.UintValue)) + } + if m.BoolValue { + n += 2 + } + return n +} + +func (m *AttrMap) Size() (n int) { + var l int + _ = l + if len(m.Attrs) > 0 { + for _, e := range m.Attrs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *QueryRequest) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + l = len(m.Query) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if len(m.Slices) > 0 { + l = 0 + for _, e := range m.Slices { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if m.Profiles { + n += 2 + } + if m.Timestamp != 0 { + n += 1 + sovInternal(uint64(m.Timestamp)) + } + l = len(m.Quantum) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Remote { + n += 2 + } + return n +} + +func (m *QueryResponse) Size() (n int) { + var l int + _ = l + l = len(m.Err) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + if len(m.Profiles) > 0 { + for _, e := range m.Profiles { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *QueryResult) Size() (n int) { + var l int + _ = l + if m.Bitmap != nil { + l = m.Bitmap.Size() + n += 1 + l + sovInternal(uint64(l)) + } + if m.N != 0 { + n += 1 + sovInternal(uint64(m.N)) + } + if len(m.Pairs) > 0 { + for _, e := range m.Pairs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + if m.Changed { + n += 2 + } + return n +} + +func (m *ImportRequest) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovInternal(uint64(m.Slice)) + } + if len(m.BitmapIDs) > 0 { + l = 0 + for _, e := range m.BitmapIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if len(m.ProfileIDs) > 0 { + l = 0 + for _, e := range m.ProfileIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + return n +} + +func (m *ImportResponse) Size() (n int) { + var l int + _ = l + l = len(m.Err) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + return n +} + +func (m *BlockDataRequest) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovInternal(uint64(m.Slice)) + } + if m.Block != 0 { + n += 1 + sovInternal(uint64(m.Block)) + } + return n +} + +func (m *BlockDataResponse) Size() (n int) { + var l int + _ = l + if len(m.BitmapIDs) > 0 { + l = 0 + for _, e := range m.BitmapIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if len(m.ProfileIDs) > 0 { + l = 0 + for _, e := range m.ProfileIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + return n +} + +func (m *Cache) Size() (n int) { + var l int + _ = l + if len(m.BitmapIDs) > 0 { + l = 0 + for _, e := range m.BitmapIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + return n +} + +func (m *SliceMaxResponse) Size() (n int) { + var l int + _ = l + if m.SliceMax != 0 { + n += 1 + sovInternal(uint64(m.SliceMax)) + } + return n +} + +func sovInternal(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozInternal(x uint64) (n int) { + return sovInternal(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *DB) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DB: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DB: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Frame) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Frame: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Bitmap) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Bitmap: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Bitmap: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Bits = append(m.Bits, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Bits = append(m.Bits, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType) + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attrs = append(m.Attrs, &Attr{}) + if err := m.Attrs[len(m.Attrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Pair) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Pair: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Pair: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + m.Key = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Key |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) + } + m.Count = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Count |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Bit) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Bit: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Bit: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapID", wireType) + } + m.BitmapID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BitmapID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProfileID", wireType) + } + m.ProfileID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProfileID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Profile) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Profile: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Profile: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + m.ID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attrs = append(m.Attrs, &Attr{}) + if err := m.Attrs[len(m.Attrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Attr) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Attr: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Attr: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + m.Type = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Type |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StringValue", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.StringValue = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UintValue", wireType) + } + m.UintValue = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UintValue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BoolValue", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BoolValue = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *AttrMap) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: AttrMap: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: AttrMap: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attrs = append(m.Attrs, &Attr{}) + if err := m.Attrs[len(m.Attrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Query", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Query = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Profiles = bool(v != 0) + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) + } + m.Timestamp = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Timestamp |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Quantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Quantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Remote", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Remote = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Err", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Err = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &QueryResult{}) + if err := m.Results[len(m.Results)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Profiles = append(m.Profiles, &Profile{}) + if err := m.Profiles[len(m.Profiles)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryResult) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Bitmap", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Bitmap == nil { + m.Bitmap = &Bitmap{} + } + if err := m.Bitmap.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field N", wireType) + } + m.N = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.N |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pairs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Pairs = append(m.Pairs, &Pair{}) + if err := m.Pairs[len(m.Pairs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Changed", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Changed = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ImportRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + } + case 5: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ImportResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Err", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Err = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockDataRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockDataRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Block", wireType) + } + m.Block = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Block |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockDataResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockDataResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + } + case 2: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Cache) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Cache: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Cache: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *SliceMaxResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: SliceMaxResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: SliceMaxResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SliceMax", wireType) + } + m.SliceMax = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SliceMax |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipInternal(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthInternal + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipInternal(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthInternal = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowInternal = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 499 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x53, 0x5d, 0x6b, 0xd4, 0x40, - 0x14, 0x65, 0x37, 0xd9, 0xcd, 0xee, 0x4d, 0x77, 0xdd, 0x1d, 0x11, 0x83, 0x50, 0x28, 0x51, 0xa4, - 0xf8, 0x50, 0xa1, 0xe8, 0x8b, 0x6f, 0x6e, 0x57, 0xa1, 0x94, 0x96, 0x7e, 0xa8, 0xcf, 0x0e, 0xeb, - 0xd8, 0x46, 0x93, 0x4c, 0x9c, 0x4c, 0xc0, 0x3e, 0xf9, 0xd7, 0x3d, 0x33, 0x99, 0xc9, 0x46, 0x58, - 0x11, 0x9f, 0x92, 0x7b, 0xe6, 0x9e, 0x7b, 0xce, 0x9c, 0xdc, 0xd0, 0xe3, 0xac, 0xd4, 0x42, 0x95, - 0x3c, 0x7f, 0xe9, 0x5f, 0x8e, 0x2a, 0x25, 0xb5, 0x64, 0x13, 0x5f, 0xa7, 0xaf, 0x69, 0xbc, 0xca, - 0x74, 0xc1, 0x2b, 0xb6, 0x47, 0x21, 0xde, 0xea, 0x64, 0x70, 0x10, 0x1c, 0x86, 0x6c, 0x9f, 0x46, - 0x6f, 0xb5, 0x56, 0x75, 0x32, 0x44, 0x19, 0x1f, 0xcf, 0x8f, 0xba, 0x09, 0x06, 0x4e, 0x53, 0x0a, - 0x2f, 0x79, 0xa6, 0x58, 0x4c, 0xc1, 0x99, 0xb8, 0x07, 0x67, 0x08, 0xce, 0x8c, 0x46, 0x27, 0xb2, - 0x29, 0x35, 0x38, 0x28, 0xd3, 0x17, 0x14, 0x60, 0x20, 0x5b, 0xd0, 0xa4, 0x55, 0x38, 0x5d, 0xbb, - 0xbe, 0x25, 0x4d, 0x2f, 0x95, 0xfc, 0x9a, 0xe5, 0x02, 0x50, 0xdb, 0xfb, 0x8a, 0x22, 0x07, 0x31, - 0xa2, 0x61, 0xd7, 0xf9, 0x0f, 0x17, 0x17, 0x14, 0x9a, 0x67, 0xdf, 0xc5, 0x94, 0x3d, 0xa4, 0xf8, - 0x46, 0xab, 0xac, 0xbc, 0xfd, 0xc4, 0xf3, 0x46, 0x80, 0x39, 0x00, 0x08, 0xc9, 0x8f, 0xe0, 0xb6, - 0x50, 0x00, 0xc8, 0xba, 0x58, 0x49, 0x99, 0xb7, 0x50, 0x08, 0x68, 0x92, 0x1e, 0x52, 0x64, 0xe6, - 0x9d, 0x23, 0x8d, 0x4e, 0x79, 0xb0, 0x53, 0xf9, 0x17, 0xed, 0x5d, 0x35, 0x42, 0xdd, 0x5f, 0x8b, - 0x1f, 0x8d, 0xa8, 0xb5, 0x31, 0xbd, 0x5e, 0x39, 0x03, 0x88, 0xc1, 0x9e, 0xd9, 0xab, 0x4d, 0xd9, - 0x9c, 0xc6, 0x37, 0x79, 0xb6, 0x11, 0x35, 0x74, 0x4d, 0xb2, 0xc8, 0xc3, 0x5d, 0xb5, 0x6e, 0x65, - 0x8d, 0x93, 0x0f, 0x59, 0x81, 0x31, 0xbc, 0xa8, 0x92, 0x11, 0xa0, 0x80, 0x3d, 0xa0, 0xe8, 0xaa, - 0xe1, 0xa5, 0x6e, 0x8a, 0x64, 0x0c, 0x60, 0x66, 0xa6, 0x5c, 0x8b, 0x42, 0x6a, 0x91, 0x44, 0xd6, - 0x6a, 0x46, 0x33, 0x67, 0xa0, 0xae, 0x64, 0x59, 0x0b, 0x93, 0xc1, 0x3b, 0xa5, 0x60, 0xc1, 0x5c, - 0xf7, 0x39, 0x45, 0x38, 0x68, 0x72, 0xed, 0x93, 0x7b, 0xb4, 0xf5, 0xef, 0x69, 0x38, 0x65, 0x4f, - 0x7b, 0x5e, 0x02, 0xdb, 0xb8, 0xdc, 0x36, 0xba, 0x93, 0xf4, 0x1b, 0xc5, 0x7d, 0xce, 0x81, 0xdf, - 0x18, 0xab, 0x15, 0x1f, 0x2f, 0xb6, 0x0c, 0xb7, 0x49, 0x53, 0x1a, 0x5c, 0xd8, 0xdc, 0xed, 0x07, - 0x34, 0x7b, 0xe2, 0xa7, 0xf7, 0x62, 0xb4, 0xeb, 0x83, 0x6b, 0x9e, 0xdc, 0xf1, 0xf2, 0x56, 0x7c, - 0x71, 0x5f, 0xe0, 0x33, 0xcd, 0x4e, 0x8b, 0x4a, 0x2a, 0xfd, 0x97, 0x60, 0xdf, 0x2b, 0x5e, 0x08, - 0x17, 0x2c, 0x4a, 0x1b, 0x2c, 0x66, 0xbb, 0xad, 0xf2, 0x7b, 0x66, 0x82, 0x35, 0x51, 0x83, 0xdd, - 0x2d, 0x5a, 0x8d, 0x64, 0x81, 0xa5, 0xfb, 0x34, 0xf7, 0x0a, 0x3b, 0x92, 0x4b, 0xcf, 0x68, 0xb1, - 0xca, 0xe5, 0xe6, 0xfb, 0x9a, 0x6b, 0xfe, 0xff, 0x1e, 0x50, 0x5a, 0x36, 0xf4, 0xcd, 0x56, 0xbf, - 0xa1, 0x65, 0x6f, 0x98, 0x93, 0xfb, 0xc3, 0xe7, 0x60, 0x87, 0xcf, 0xa1, 0xf5, 0xf9, 0x04, 0x3f, - 0x13, 0xdf, 0xdc, 0xed, 0xea, 0x4f, 0x9f, 0xd1, 0xc2, 0xaa, 0x9e, 0xf3, 0x9f, 0xdd, 0x58, 0xac, - 0x95, 0xc7, 0xda, 0x9f, 0xe7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x12, 0x50, 0x70, 0xfe, - 0x03, 0x00, 0x00, + // 638 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x54, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0x66, 0x63, 0x3b, 0x3f, 0x93, 0x36, 0x4a, 0x57, 0x01, 0x59, 0x15, 0x8a, 0xa2, 0x15, 0xa0, + 0x70, 0x20, 0x95, 0xca, 0x03, 0x54, 0xb8, 0x29, 0x52, 0x84, 0x5a, 0xb5, 0xdb, 0xc2, 0x7d, 0x69, + 0x97, 0xd6, 0xc2, 0x7f, 0xac, 0xd7, 0x12, 0x3d, 0x72, 0xe0, 0x00, 0x4f, 0xc0, 0xa3, 0xf0, 0x08, + 0x1c, 0x79, 0x04, 0x54, 0x5e, 0x04, 0xed, 0x9f, 0x6d, 0x84, 0x08, 0x3d, 0x70, 0xdb, 0xef, 0x1b, + 0xef, 0xce, 0x37, 0xf3, 0xcd, 0x18, 0x46, 0x71, 0x26, 0xb9, 0xc8, 0x58, 0xb2, 0x28, 0x44, 0x2e, + 0x73, 0xdc, 0x77, 0x98, 0x3c, 0x82, 0xce, 0x32, 0xc2, 0x33, 0x18, 0x9e, 0xc5, 0x29, 0x3f, 0xa9, + 0x58, 0x26, 0xab, 0x34, 0x44, 0x33, 0x34, 0x1f, 0xd0, 0x36, 0x45, 0x1e, 0x43, 0xf0, 0x5c, 0xb0, + 0x94, 0xdf, 0xe2, 0xd3, 0x08, 0xba, 0x51, 0x2c, 0x53, 0x56, 0x60, 0x0c, 0x7e, 0x14, 0xcb, 0x32, + 0x44, 0x33, 0x6f, 0xee, 0x53, 0x7d, 0xc6, 0x0f, 0x20, 0x78, 0x26, 0xa5, 0x28, 0xc3, 0xce, 0xcc, + 0x9b, 0x0f, 0x77, 0x47, 0x8b, 0x5a, 0x9a, 0xa2, 0xa9, 0x09, 0x92, 0x05, 0xf8, 0xc7, 0x2c, 0x16, + 0x78, 0x0c, 0xde, 0x0b, 0x7e, 0xad, 0xb3, 0xf8, 0x54, 0x1d, 0xf1, 0x04, 0x82, 0xfd, 0xbc, 0xca, + 0x64, 0xd8, 0xd1, 0x9c, 0x01, 0x64, 0x0f, 0xbc, 0x28, 0x96, 0x78, 0x1b, 0xfa, 0x26, 0xf5, 0x6a, + 0x69, 0xef, 0xd4, 0x18, 0xdf, 0x87, 0xc1, 0xb1, 0xc8, 0xdf, 0xc4, 0x09, 0x5f, 0x2d, 0xed, 0xe5, + 0x86, 0x20, 0x7b, 0xd0, 0xb3, 0x00, 0x8f, 0xa0, 0x53, 0x5f, 0xef, 0xac, 0x96, 0xb7, 0x54, 0xfc, + 0x19, 0x81, 0xaf, 0x4e, 0x6d, 0xc9, 0x03, 0x23, 0x19, 0x83, 0x7f, 0x76, 0x5d, 0x70, 0x9b, 0x54, + 0x9f, 0x55, 0x1b, 0x4f, 0xa5, 0x88, 0xb3, 0xcb, 0x57, 0x2c, 0xa9, 0x78, 0xe8, 0x99, 0x36, 0xb6, + 0x28, 0xa5, 0xf7, 0x65, 0x9c, 0x49, 0x13, 0xf7, 0x8d, 0xde, 0x9a, 0x50, 0xd1, 0x28, 0xcf, 0x13, + 0x13, 0x0d, 0x66, 0x68, 0xde, 0xa7, 0x0d, 0x41, 0x76, 0xa0, 0xa7, 0xb4, 0x1c, 0xb2, 0xa2, 0x51, + 0x8f, 0xd6, 0xa9, 0xff, 0x8a, 0x60, 0xe3, 0xa4, 0xe2, 0xe2, 0x9a, 0xf2, 0x77, 0x15, 0x2f, 0xa5, + 0x6a, 0xc2, 0x32, 0xb2, 0x45, 0xa8, 0x09, 0x99, 0x40, 0xa0, 0xe3, 0xba, 0x88, 0x01, 0x35, 0x00, + 0xdf, 0x83, 0xee, 0x69, 0x12, 0x9f, 0xf3, 0x32, 0xf4, 0xb4, 0xc5, 0x16, 0x29, 0x1f, 0x6c, 0x37, + 0x4b, 0x2d, 0xbd, 0x4f, 0x6b, 0xac, 0x94, 0xab, 0x69, 0x29, 0x25, 0x4b, 0x0b, 0xad, 0xdc, 0xa3, + 0x0d, 0x81, 0x43, 0xe8, 0xb9, 0xd1, 0xea, 0xea, 0x4c, 0x0e, 0xaa, 0x5c, 0x94, 0xa7, 0xb9, 0xe4, + 0x61, 0x4f, 0xbf, 0x68, 0x11, 0xf9, 0x80, 0x60, 0xd3, 0x4a, 0x2f, 0x8b, 0x3c, 0x2b, 0xb9, 0x72, + 0xe0, 0x40, 0x08, 0xe7, 0xc0, 0x81, 0x10, 0x78, 0x07, 0x7a, 0x94, 0x97, 0x55, 0x22, 0x9d, 0x89, + 0x77, 0x9b, 0x36, 0xb8, 0xbb, 0x55, 0x22, 0xa9, 0xfb, 0x0a, 0x3f, 0x69, 0x15, 0xe0, 0xe9, 0x1b, + 0x5b, 0xcd, 0x0d, 0x1b, 0x69, 0x6a, 0x22, 0x1f, 0x11, 0x0c, 0x5b, 0xef, 0xe0, 0xb9, 0x5b, 0x01, + 0x2d, 0x62, 0xb8, 0x3b, 0x6e, 0x2e, 0x1b, 0x9e, 0xba, 0x15, 0xd9, 0x00, 0x74, 0x64, 0x07, 0x03, + 0x1d, 0x29, 0xb3, 0xd4, 0xd8, 0xbb, 0x9c, 0x2d, 0xb3, 0x14, 0x4d, 0x4d, 0x50, 0xf5, 0x68, 0xff, + 0x8a, 0x65, 0x97, 0xfc, 0xc2, 0x36, 0xd7, 0x41, 0xf2, 0x09, 0xc1, 0xe6, 0x2a, 0x2d, 0x72, 0x21, + 0xd7, 0xf8, 0xa8, 0xf7, 0xd8, 0xf9, 0x68, 0x96, 0x7a, 0x02, 0x81, 0x76, 0x4e, 0xcf, 0xa1, 0x4f, + 0x0d, 0xd0, 0x33, 0x66, 0xb7, 0x47, 0xd9, 0xa8, 0x0c, 0x6e, 0x08, 0x3c, 0x05, 0xa8, 0xd7, 0xa7, + 0x0c, 0x03, 0x1d, 0x6e, 0x31, 0x84, 0xc0, 0xc8, 0x49, 0xf9, 0x9b, 0x2f, 0xe4, 0x02, 0xc6, 0x51, + 0x92, 0x9f, 0xbf, 0x5d, 0x32, 0xc9, 0xfe, 0x87, 0xe2, 0x09, 0x04, 0xfa, 0x3d, 0xbb, 0x2f, 0x06, + 0x90, 0x13, 0xd8, 0x6a, 0x65, 0xb1, 0x62, 0x7e, 0x2b, 0x0e, 0xad, 0x2f, 0xae, 0xf3, 0x47, 0x71, + 0x0f, 0x21, 0xd8, 0x67, 0xe7, 0x57, 0xff, 0x78, 0x86, 0x2c, 0x60, 0xac, 0x85, 0x1d, 0xb2, 0xf7, + 0x75, 0xe2, 0x6d, 0xe8, 0x3b, 0xce, 0xfd, 0xa3, 0x1c, 0x8e, 0xc6, 0xdf, 0x6e, 0xa6, 0xe8, 0xfb, + 0xcd, 0x14, 0xfd, 0xb8, 0x99, 0xa2, 0x2f, 0x3f, 0xa7, 0x77, 0x5e, 0x77, 0xf5, 0x0f, 0xfb, 0xe9, + 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x54, 0x5a, 0xc6, 0x13, 0xc2, 0x05, 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index 79e1d627d..3c426e6eb 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -1,30 +1,41 @@ +syntax = "proto3"; + package internal; +message DB { + string TimeQuantum = 1; +} + +message Frame { + string TimeQuantum = 1; +} + message Bitmap { - repeated uint64 Bits = 1; - repeated Attr Attrs = 2; + repeated uint64 Bits = 1; + repeated Attr Attrs = 2; } message Pair { - required uint64 Key = 1; - required uint64 Count = 2; + uint64 Key = 1; + uint64 Count = 2; } message Bit { - required uint64 BitmapID = 1; - required uint64 ProfileID = 2; + uint64 BitmapID = 1; + uint64 ProfileID = 2; } message Profile { - required uint64 ID = 1; + uint64 ID = 1; repeated Attr Attrs = 2; } message Attr { - required string Key = 1; - optional string StringValue = 2; - optional uint64 UintValue = 3; - optional bool BoolValue = 4; + string Key = 1; + uint64 Type = 2; + string StringValue = 3; + uint64 UintValue = 4; + bool BoolValue = 5; } message AttrMap { @@ -32,49 +43,49 @@ message AttrMap { } message QueryRequest { - required string DB = 1; - required string Query = 2; - repeated uint64 Slices = 3; - optional bool Profiles = 4; - optional int64 Timestamp = 5; - optional uint32 Quantum = 6; - optional bool Remote = 7; + string DB = 1; + string Query = 2; + repeated uint64 Slices = 3; + bool Profiles = 4; + int64 Timestamp = 5; + string Quantum = 6; + bool Remote = 7; } message QueryResponse { - optional string Err = 1; - repeated QueryResult Results = 2; - repeated Profile Profiles = 3; + string Err = 1; + repeated QueryResult Results = 2; + repeated Profile Profiles = 3; } message QueryResult { - optional Bitmap Bitmap = 1; - optional uint64 N = 2; - repeated Pair Pairs = 3; - optional bool Changed = 4; + Bitmap Bitmap = 1; + uint64 N = 2; + repeated Pair Pairs = 3; + bool Changed = 4; } message ImportRequest { - required string DB = 1; - required string Frame = 2; - required uint64 Slice = 3; - repeated uint64 BitmapIDs = 4; + string DB = 1; + string Frame = 2; + uint64 Slice = 3; + repeated uint64 BitmapIDs = 4; repeated uint64 ProfileIDs = 5; } message ImportResponse { - optional string Err = 1; + string Err = 1; } message BlockDataRequest { - required string DB = 1; - required string Frame = 2; - required uint64 Slice = 3; - required uint64 Block = 4; + string DB = 1; + string Frame = 2; + uint64 Slice = 3; + uint64 Block = 4; } message BlockDataResponse { - repeated uint64 BitmapIDs = 1; + repeated uint64 BitmapIDs = 1; repeated uint64 ProfileIDs = 2; } @@ -83,5 +94,6 @@ message Cache { } message SliceMaxResponse { - required uint64 SliceMax = 1; + uint64 SliceMax = 1; } + diff --git a/pilosa.go b/pilosa.go index 4d1fbba41..d6c07bb0c 100644 --- a/pilosa.go +++ b/pilosa.go @@ -3,12 +3,9 @@ package pilosa import ( "errors" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) -//go:generate protoc --gogo_out=. internal/internal.proto - var ( // ErrHostRequired is returned when excuting a remote operation without a host. ErrHostRequired = errors.New("host required") @@ -57,7 +54,7 @@ func decodeProfiles(a []*internal.Profile) []*Profile { // encodeProfile converts p into its internal representation. func encodeProfile(p *Profile) *internal.Profile { return &internal.Profile{ - ID: proto.Uint64(p.ID), + ID: p.ID, Attrs: encodeAttrs(p.Attrs), } } @@ -65,12 +62,12 @@ func encodeProfile(p *Profile) *internal.Profile { // decodeProfile converts b from its internal representation. func decodeProfile(pb *internal.Profile) *Profile { p := &Profile{ - ID: pb.GetID(), + ID: pb.ID, } - if len(pb.GetAttrs()) > 0 { - p.Attrs = make(map[string]interface{}, len(pb.GetAttrs())) - for _, attr := range pb.GetAttrs() { + if len(pb.Attrs) > 0 { + p.Attrs = make(map[string]interface{}, len(pb.Attrs)) + for _, attr := range pb.Attrs { k, v := decodeAttr(attr) p.Attrs[k] = v } diff --git a/server.go b/server.go index 635ed0e10..670ed707d 100644 --- a/server.go +++ b/server.go @@ -255,6 +255,5 @@ func checkMaxSlice(hostport string) (uint64, error) { return 0, err } - return *pb.SliceMax, nil - + return pb.SliceMax, nil } diff --git a/time.go b/time.go index ce11ea1d9..b117023ad 100644 --- a/time.go +++ b/time.go @@ -2,122 +2,166 @@ package pilosa import ( "errors" + "fmt" "strings" "time" ) -// TimeQuantum represents a time granularity for time-based bitmap ids. -type TimeQuantum uint +// ErrInvalidTimeQuantum is returned when parsing a time quantum. +var ErrInvalidTimeQuantum = errors.New("invalid time quantum") -// ParseTimeQuantum parses s into a quantum. -func ParseTimeQuantum(s string) (TimeQuantum, error) { - switch strings.ToUpper(s) { - case "Y": - return Y, nil - case "M": - return YM, nil - case "D": - return YMD, nil - case "H": - return YMDH, nil - default: - return 0, errors.New("invalid quantum") - } -} +// TimeQuantum represents a time granularity for time-based bitmaps. +type TimeQuantum string -// String returns the string representation of the quantum. -func (q TimeQuantum) String() string { +// HasYear returns true if the quantum contains a 'Y' unit. +func (q TimeQuantum) HasYear() bool { return strings.ContainsRune(string(q), 'Y') } + +// HasMonth returns true if the quantum contains a 'M' unit. +func (q TimeQuantum) HasMonth() bool { return strings.ContainsRune(string(q), 'M') } + +// HasDay returns true if the quantum contains a 'D' unit. +func (q TimeQuantum) HasDay() bool { return strings.ContainsRune(string(q), 'D') } + +// HasHour returns true if the quantum contains a 'H' unit. +func (q TimeQuantum) HasHour() bool { return strings.ContainsRune(string(q), 'H') } + +// Valid returns true if q is a valid time quantum value. +func (q TimeQuantum) Valid() bool { switch q { - case Y: - return "Y" - case YM: - return "M" - case YMD: - return "D" - case YMDH: - return "H" + case "Y", "YM", "YMD", "YMDH", + "M", "MD", "MDH", + "D", "DH", + "H", + "": + return true default: - return "Y" + return false } } -const ( - Y TimeQuantum = 3 - YM TimeQuantum = 2 - YMD TimeQuantum = 1 - YMDH TimeQuantum = 0 -) - -// TimeID returns a packed time identifier from a year/month/day/hour & tile. -func TimeID(q TimeQuantum, year uint, month uint, day uint, hour uint, tileID uint64) uint64 { - v := uint64((uint(q) << 30) | ((year - 1970) << 23) | (month << 19) | (day << 14) | (hour << 9)) - return (v << 32) | tileID +// ParseTimeQuantum parses v into a time quantum. +func ParseTimeQuantum(v string) (TimeQuantum, error) { + q := TimeQuantum(strings.ToUpper(v)) + if !q.Valid() { + return "", ErrInvalidTimeQuantum + } + return q, nil } -// TimeIDsFromRange returns timestamp bitmap ids within a time range. -func TimeIDsFromRange(start, end time.Time, tileID uint64) []uint64 { +// FrameByTimeUnit returns the frame name for time with a given quantum unit. +func FrameByTimeUnit(name string, t time.Time, unit rune) string { + switch unit { + case 'Y': + return fmt.Sprintf("%s_%s", name, t.Format("2006")) + case 'M': + return fmt.Sprintf("%s_%s", name, t.Format("200601")) + case 'D': + return fmt.Sprintf("%s_%s", name, t.Format("20060102")) + case 'H': + return fmt.Sprintf("%s_%s", name, t.Format("2006010215")) + default: + return "" + } +} + +// FramesByTime returns a list of frames for a given timestamp. +func FramesByTime(name string, t time.Time, q TimeQuantum) []string { + a := make([]string, 0, len(q)) + for _, unit := range q { + frame := FrameByTimeUnit(name, t, unit) + if frame == "" { + continue + } + a = append(a, frame) + } + return a +} + +// FramesByTimeRange returns a list of frames to traverse to query a time range. +func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start - var results []uint64 - for t.Before(end) { - if !nextDay(t, end) { + // Save flags for performance. + hasYear := q.HasYear() + hasMonth := q.HasMonth() + hasDay := q.HasDay() + hasHour := q.HasHour() + + var results []string + + // Walk up from smallest units to largest units. + if hasHour || hasDay || hasMonth { + for t.Before(end) { + if hasHour { + if !nextDayGTE(t, end) { + break + } else if t.Hour() != 0 { + results = append(results, FrameByTimeUnit(name, t, 'H')) + t = t.Add(time.Hour) + continue + } + + } + + if hasDay { + if !nextMonthGTE(t, end) { + break + } else if t.Day() != 1 { + results = append(results, FrameByTimeUnit(name, t, 'D')) + t = t.AddDate(0, 0, 1) + continue + } + } + + if hasMonth { + if !nextYearGTE(t, end) { + break + } else if t.Month() != 1 { + results = append(results, FrameByTimeUnit(name, t, 'M')) + t = t.AddDate(0, 1, 0) + continue + } + } + + // If a unit exists but isn't set and there are no larger units + // available then we need to exit the loop because we are no longer + // making progress. break } - if t.Hour() == 0 { - if !nextMonth(t, end) { - break - } - - if t.Day() == 1 { - if !nextYear(t, end) { - break - } - - if t.Month() == 1 { - break - } - - results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) - t = t.AddDate(0, 1, 0) - } else { - results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) - t = t.AddDate(0, 0, 1) - } - } else { - results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) - t = t.Add(time.Hour) - } } + // Walk back down from largest units to smallest units. for t.Before(end) { - if nextYear(t, end) { - results = append(results, TimeID(Y, uint(t.Year()), 0, 0, 0, tileID)) + if hasYear && nextYearGTE(t, end) { + results = append(results, FrameByTimeUnit(name, t, 'Y')) t = t.AddDate(1, 0, 0) - } else if nextMonth(t, end) { - results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) + } else if hasMonth && nextMonthGTE(t, end) { + results = append(results, FrameByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) - } else if nextDay(t, end) { - results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) + } else if hasDay && nextDayGTE(t, end) { + results = append(results, FrameByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) - } else { - results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) + } else if hasHour { + results = append(results, FrameByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) + } else { + break } } return results } -func nextYear(start time.Time, end time.Time) bool { - next := start.AddDate(1, 0, 0) +func nextYearGTE(t time.Time, end time.Time) bool { + next := t.AddDate(1, 0, 0) if next.Year() == end.Year() { return true } return end.After(next) } -func nextMonth(start time.Time, end time.Time) bool { - next := start.AddDate(0, 1, 0) +func nextMonthGTE(t time.Time, end time.Time) bool { + next := t.AddDate(0, 1, 0) y1, m1, _ := next.Date() y2, m2, _ := end.Date() if (y1 == y2) && (m1 == m2) { @@ -126,8 +170,8 @@ func nextMonth(start time.Time, end time.Time) bool { return end.After(next) } -func nextDay(start time.Time, end time.Time) bool { - next := start.AddDate(0, 0, 1) +func nextDayGTE(t time.Time, end time.Time) bool { + next := t.AddDate(0, 0, 1) y1, m1, d1 := next.Date() y2, m2, d2 := end.Date() if (y1 == y2) && (m1 == m2) && (d1 == d2) { @@ -135,23 +179,3 @@ func nextDay(start time.Time, end time.Time) bool { } return end.After(next) } - -// TimeIDsFromQuantum returns a list of time identifiers for a single quantum. -func TimeIDsFromQuantum(q TimeQuantum, t time.Time, tileID uint64) []uint64 { - y, m, d, h := uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()) - - v := make([]uint64, 0, 4) - if q <= Y { - v = append(v, TimeID(Y, y, 0, 0, 0, tileID)) - } - if q <= YM { - v = append(v, TimeID(YM, y, m, 0, 0, tileID)) - } - if q <= YMD { - v = append(v, TimeID(YMD, y, m, d, 0, tileID)) - } - if q <= YMDH { - v = append(v, TimeID(YMDH, y, m, d, h, tileID)) - } - return v -} diff --git a/time_test.go b/time_test.go index aed62849c..28d86555d 100644 --- a/time_test.go +++ b/time_test.go @@ -1,134 +1,162 @@ package pilosa_test import ( + "reflect" "testing" "time" "github.com/pilosa/pilosa" ) -func TestTimeIDsFromRange_1h_0(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-08-11 14:00"), - *MustParseTime("2014-08-11 16:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } +// Ensure string can be parsed into time quantum. +func TestParseTimeQuantum(t *testing.T) { + t.Run("OK", func(t *testing.T) { + if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil { + t.Fatalf("unexpected error: %s", err) + } else if q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %#v", q) + } + }) + + t.Run("ErrInvalidTimeQuantum", func(t *testing.T) { + if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum { + t.Fatalf("unexpected error: %s", err) + } + }) } -func TestTimeIDsFromRange_1h_1(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 10:03"), - *MustParseTime("2014-01-02 11:03"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } +// Ensure generated frame name can be returned for a given time unit. +func TestFrameByTimeUnit(t *testing.T) { + ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) + + t.Run("Y", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'Y'); s != "F_2000" { + t.Fatalf("unexpected name: %s", s) + } + }) + t.Run("M", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'M'); s != "F_200001" { + t.Fatalf("unexpected name: %s", s) + } + }) + t.Run("D", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'D'); s != "F_20000102" { + t.Fatalf("unexpected name: %s", s) + } + }) + t.Run("H", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + t.Fatalf("unexpected name: %s", s) + } + }) } -func TestTimeIDsFromRange_2h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 10:03"), - *MustParseTime("2014-01-02 12:03"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } +// Ensure all applicable frame names can be generated when mutating a time bit. +func TestFramesByTime(t *testing.T) { + ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) + + t.Run("YMDH", func(t *testing.T) { + a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("YMDH")) + if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) { + t.Fatalf("unexpected names: %+v", a) + } + }) + + t.Run("D", func(t *testing.T) { + a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("D")) + if !reflect.DeepEqual(a, []string{"F_20000102"}) { + t.Fatalf("unexpected names: %+v", a) + } + }) } -func TestTimeIDsFromRange_24h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 12:03"), - *MustParseTime("2014-01-03 12:03"), - uint64(1), - ); len(m) != 24 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1d(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 00:00"), - *MustParseTime("2014-01-03 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1d1h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 00:00"), - *MustParseTime("2014-01-03 01:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 23:00"), - *MustParseTime("2014-01-04 00:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d1h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 23:00"), - *MustParseTime("2014-01-04 01:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1y(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-01 00:00"), - *MustParseTime("2015-01-01 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d1m(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-30 23:00"), - *MustParseTime("2014-03-01 00:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d1m1d1h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-30 23:00"), - *MustParseTime("2014-03-02 01:00"), - uint64(1), - ); len(m) != 5 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromQuantum(t *testing.T) { - _ = pilosa.TimeIDsFromQuantum(pilosa.YMD, *MustParseTime("1970-01-01 00:00"), uint64(15027)) +// Ensure sets of frames can be returned for a given time range. +func TestFramesByTimeRange(t *testing.T) { + t.Run("Y", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) + if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("YM", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) + if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("YMD", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) + if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("YMDH", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) + if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("M", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) + if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("MD", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) + if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("MDH", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) + if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("D", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) + if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("DH", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) + if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("H", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) + if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) } // DefaultTimeLayout is the time layout used by the tests. const DefaultTimeLayout = "2006-01-02 15:04" // MustParseTime parses value using DefaultTimeLayout. Panic on error. -func MustParseTime(value string) *time.Time { +func MustParseTime(value string) time.Time { v, err := time.Parse(DefaultTimeLayout, value) if err != nil { panic(err) } + return v +} + +// MustParseTimePtr parses value using DefaultTimeLayout. Panic on error. +func MustParseTimePtr(value string) *time.Time { + v := MustParseTime(value) return &v } + +// MustParseTimeQuantum parses v into a time quantum. Panic on error. +func MustParseTimeQuantum(v string) pilosa.TimeQuantum { + q, err := pilosa.ParseTimeQuantum(v) + if err != nil { + panic(err) + } + return q +} From c1bbd338b343df5a92e0690587a60c55af9c8b1b Mon Sep 17 00:00:00 2001 From: Travis Date: Mon, 12 Dec 2016 12:36:21 -0600 Subject: [PATCH 06/11] Fixes broken index sync test. --- frame.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/frame.go b/frame.go index b215d80d2..0456488a3 100644 --- a/frame.go +++ b/frame.go @@ -218,7 +218,7 @@ func (f *Frame) SetTimeQuantum(q TimeQuantum) error { // Update value on frame. f.timeQuantum = q - // Perist meta data to disk. + // Persist meta data to disk. if err := f.saveMeta(); err != nil { return err } @@ -288,7 +288,7 @@ func (f *Frame) newFragment(path string, slice uint64) *Fragment { // SetBit sets a bit within the frame. func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) { - slice := bitmapID / SliceWidth + slice := profileID / SliceWidth frag, err := f.CreateFragmentIfNotExists(slice) if err != nil { return changed, err From f53cdd32750964c01a41e56f8e3fd0a89ebae608 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Thu, 8 Dec 2016 12:29:49 -0700 Subject: [PATCH 07/11] Add time-based frames and Range() support. --- attr.go | 40 +- bitmap.go | 4 +- cache.go | 9 +- client.go | 24 +- client_test.go | 6 +- db.go | 149 ++ db_test.go | 68 + executor.go | 52 +- executor_test.go | 72 +- fragment.go | 44 +- fragment_test.go | 46 +- frame.go | 97 +- frame_test.go | 35 + handler.go | 182 +- handler_test.go | 101 +- index_test.go | 44 +- internal/internal.go | 2 + internal/internal.pb.go | 3954 +++++++++++++++++++++++++++++++++++---- internal/internal.proto | 83 +- pilosa.go | 13 +- server.go | 1 - time.go | 228 ++- time_test.go | 242 +-- 23 files changed, 4661 insertions(+), 835 deletions(-) diff --git a/attr.go b/attr.go index dacbf9bbb..1425676cf 100644 --- a/attr.go +++ b/attr.go @@ -17,6 +17,13 @@ import ( // AttrBlockSize is the size of attribute blocks for anti-entropy. const AttrBlockSize = 100 +// Attribute data type enum. +const ( + AttrTypeString = 1 + AttrTypeUint = 2 + AttrTypeBool = 3 +) + // AttrStore represents a storage layer for attributes. type AttrStore struct { mu sync.Mutex @@ -303,32 +310,39 @@ func decodeAttrs(pb []*internal.Attr) map[string]interface{} { // encodeAttr converts a key/value pair into an Attr internal representation. func encodeAttr(key string, value interface{}) *internal.Attr { - pb := &internal.Attr{Key: proto.String(key)} + pb := &internal.Attr{Key: key} switch value := value.(type) { case string: - pb.StringValue = proto.String(value) + pb.Type = AttrTypeString + pb.StringValue = value case float64: - pb.UintValue = proto.Uint64(uint64(value)) + pb.Type = AttrTypeUint + pb.UintValue = uint64(value) case uint64: - pb.UintValue = proto.Uint64(value) + pb.Type = AttrTypeUint + pb.UintValue = value case int64: - pb.UintValue = proto.Uint64(uint64(value)) + pb.Type = AttrTypeUint + pb.UintValue = uint64(value) case bool: - pb.BoolValue = proto.Bool(value) + pb.Type = AttrTypeBool + pb.BoolValue = value } return pb } // decodeAttr converts from an Attr internal representation to a key/value pair. func decodeAttr(attr *internal.Attr) (key string, value interface{}) { - if attr.StringValue != nil { - return attr.GetKey(), attr.GetStringValue() - } else if attr.UintValue != nil { - return attr.GetKey(), attr.GetUintValue() - } else if attr.BoolValue != nil { - return attr.GetKey(), attr.GetBoolValue() + switch attr.Type { + case AttrTypeString: + return attr.Key, attr.StringValue + case AttrTypeUint: + return attr.Key, attr.UintValue + case AttrTypeBool: + return attr.Key, attr.BoolValue + default: + return attr.Key, nil } - return attr.GetKey(), nil } // cloneAttrs returns a shallow clone of m. diff --git a/bitmap.go b/bitmap.go index b3d8d0096..55664b24b 100644 --- a/bitmap.go +++ b/bitmap.go @@ -227,8 +227,8 @@ func decodeBitmap(pb *internal.Bitmap) *Bitmap { } b := NewBitmap() - b.Attrs = decodeAttrs(pb.GetAttrs()) - for _, v := range pb.GetBits() { + b.Attrs = decodeAttrs(pb.Attrs) + for _, v := range pb.Bits { b.SetBit(v) } return b diff --git a/cache.go b/cache.go index 4efce34f0..a1cba3dec 100644 --- a/cache.go +++ b/cache.go @@ -7,7 +7,6 @@ import ( "sort" "time" - "github.com/gogo/protobuf/proto" "github.com/golang/groupcache/lru" "github.com/pilosa/pilosa/internal" ) @@ -216,15 +215,15 @@ type Pair struct { func encodePair(p Pair) *internal.Pair { return &internal.Pair{ - Key: proto.Uint64(p.Key), - Count: proto.Uint64(p.Count), + Key: p.Key, + Count: p.Count, } } func decodePair(pb *internal.Pair) Pair { return Pair{ - Key: pb.GetKey(), - Count: pb.GetCount(), + Key: pb.Key, + Count: pb.Count, } } diff --git a/client.go b/client.go index ed6a1a2ab..47169571b 100644 --- a/client.go +++ b/client.go @@ -151,9 +151,9 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire // Encode query request. buf, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String(db), - Query: proto.String(query), - Remote: proto.Bool(!allowRedirect), + DB: db, + Query: query, + Remote: !allowRedirect, }) if err != nil { return nil, fmt.Errorf("marshal: %s", err) @@ -187,7 +187,7 @@ func (c *Client) ExecuteQuery(ctx context.Context, db, query string, allowRedire var qresp internal.QueryResponse if err := proto.Unmarshal(body, &qresp); err != nil { return nil, fmt.Errorf("unmarshal response: %s", err) - } else if s := qresp.GetErr(); s != "" { + } else if s := qresp.Err; s != "" { return nil, errors.New(s) } @@ -230,9 +230,9 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e // Marshal bits to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ - DB: proto.String(db), - Frame: proto.String(frame), - Slice: proto.Uint64(slice), + DB: db, + Frame: frame, + Slice: slice, BitmapIDs: bitmapIDs, ProfileIDs: profileIDs, }) @@ -272,7 +272,7 @@ func (c *Client) importNode(ctx context.Context, node *Node, buf []byte) error { var isresp internal.ImportResponse if err := proto.Unmarshal(body, &isresp); err != nil { return fmt.Errorf("unmarshal import response: %s", err) - } else if s := isresp.GetErr(); s != "" { + } else if s := isresp.Err; s != "" { return errors.New(s) } @@ -644,10 +644,10 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame string, slice uin // BlockData returns bitmap/profile id pairs for a block. func (c *Client) BlockData(ctx context.Context, db, frame string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ - DB: proto.String(db), - Frame: proto.String(frame), - Slice: proto.Uint64(slice), - Block: proto.Uint64(uint64(block)), + DB: db, + Frame: frame, + Slice: slice, + Block: uint64(block), }) if err != nil { return nil, nil, err diff --git a/client_test.go b/client_test.go index d81e42a72..f37fea6ee 100644 --- a/client_test.go +++ b/client_test.go @@ -96,11 +96,11 @@ func TestClient_FragmentBlocks(t *testing.T) { defer idx.Close() // Set two bits on blocks 0 & 3. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(pilosa.HashBlockSize*3, 100, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(pilosa.HashBlockSize*3, 100) // Set a bit on a different slice. - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, 1, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, 1) s := NewServer() defer s.Close() diff --git a/db.go b/db.go index 056214f7d..6178cfc95 100644 --- a/db.go +++ b/db.go @@ -3,10 +3,15 @@ package pilosa import ( "errors" "fmt" + "io/ioutil" "os" "path/filepath" "sort" "sync" + "time" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) // DB represents a container for frames. @@ -15,6 +20,10 @@ type DB struct { path string name string + // Default time quantum for all frames in database. + // This can be overridden by individual frames. + timeQuantum TimeQuantum + // Frames by name. frames map[string]*Frame @@ -57,6 +66,11 @@ func (db *DB) Open() error { return err } + // Read meta file. + if err := db.loadMeta(); err != nil { + return err + } + if err := db.openFrames(); err != nil { return err } @@ -97,6 +111,45 @@ func (db *DB) openFrames() error { return nil } +// loadMeta reads meta data for the database, if any. +func (db *DB) loadMeta() error { + var pb internal.DB + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(db.path, "meta")) + if os.IsNotExist(err) { + db.timeQuantum = "" + return nil + } else if err != nil { + return err + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + } + + // Copy metadata fields. + db.timeQuantum = TimeQuantum(pb.TimeQuantum) + + return nil +} + +// saveMeta writes meta data for the database. +func (db *DB) saveMeta() error { + // Marshal metadata. + buf, err := proto.Marshal(&internal.DB{TimeQuantum: string(db.timeQuantum)}) + if err != nil { + return err + } + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(db.path, "meta"), buf, 0666); err != nil { + return err + } + + return nil +} + // Close closes the database and its frames. func (db *DB) Close() error { db.mu.Lock() @@ -133,6 +186,34 @@ func (db *DB) MaxSlice() uint64 { return max } +// TimeQuantum returns the default time quantum for the database. +func (db *DB) TimeQuantum() TimeQuantum { + db.mu.Lock() + defer db.mu.Unlock() + return db.timeQuantum +} + +// SetTimeQuantum sets the default time quantum for the database. +func (db *DB) SetTimeQuantum(q TimeQuantum) error { + db.mu.Lock() + defer db.mu.Unlock() + + // Validate input. + if !q.Valid() { + return ErrInvalidTimeQuantum + } + + // Update value on database. + db.timeQuantum = q + + // Perist meta data to disk. + if err := db.saveMeta(); err != nil { + return err + } + + return nil +} + // FramePath returns the path to a frame in the database. func (db *DB) FramePath(name string) string { return filepath.Join(db.path, name) } @@ -194,6 +275,74 @@ func (db *DB) newFrame(path, name string) *Frame { return f } +// DeleteFrame removes a frame from the database. +func (db *DB) DeleteFrame(name string) error { + db.mu.Lock() + defer db.mu.Unlock() + + // Ignore if frame doesn't exist. + f := db.frame(name) + if f == nil { + return nil + } + + // Close frame. + if err := f.Close(); err != nil { + return err + } + + // Delete frame directory. + if err := os.RemoveAll(db.FramePath(name)); err != nil { + return err + } + + // Remove reference. + delete(db.frames, name) + + db.stats.Count("frameN", -1) + + return nil +} + +// SetBit sets a bit for a given profile & bitmap. +// If a timestamp is specified then set all bits for the different quantum units. +func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) { + // Read frame. + f, err := db.CreateFrameIfNotExists(name) + if err != nil { + return changed, err + } + + // If this is a non-time bit then simply set the bit on the frame. + if t == nil { + return f.SetBit(bitmapID, profileID) + } + + // Determine quantum of frame. Set to the default quantum if it is unset. + q := f.TimeQuantum() + if q == "" { + q = db.TimeQuantum() + if err := f.SetTimeQuantum(q); err != nil { + return changed, err + } + } + + // If a timestamp is specified then set bits across all frames for the quantum. + for _, subname := range FramesByTime(name, *t, q) { + f, err := db.CreateFrameIfNotExists(subname) + if err != nil { + return changed, err + } + + if c, err := f.SetBit(bitmapID, profileID); err != nil { + return changed, err + } else if c { + changed = true + } + } + return changed, nil +} + type dbSlice []*DB func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/db_test.go b/db_test.go index 5ffc68392..46a138c87 100644 --- a/db_test.go +++ b/db_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "os" "testing" + "time" "github.com/pilosa/pilosa" ) @@ -34,6 +35,49 @@ func TestDB_CreateFrameIfNotExists(t *testing.T) { } } +// Ensure database can delete a frame. +func TestDB_DeleteFrame(t *testing.T) { + db := MustOpenDB() + defer db.Close() + + // Create frame. + if _, err := db.CreateFrameIfNotExists("f"); err != nil { + t.Fatal(err) + } + + // Delete frame & verify it's gone. + if err := db.DeleteFrame("f"); err != nil { + t.Fatal(err) + } else if db.Frame("f") != nil { + t.Fatal("expected nil frame") + } + + // Delete again to make sure it doesn't error. + if err := db.DeleteFrame("f"); err != nil { + t.Fatal(err) + } +} + +// Ensure database can set the default time quantum. +func TestDB_SetTimeQuantum(t *testing.T) { + db := MustOpenDB() + defer db.Close() + + // Set & retrieve time quantum. + if err := db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload database and verify that it is persisted. + if err := db.Reopen(); err != nil { + t.Fatal(err) + } else if q := db.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} + // DB represents a test wrapper for pilosa.DB. type DB struct { *pilosa.DB @@ -63,3 +107,27 @@ func (db *DB) Close() error { defer os.RemoveAll(db.Path()) return db.DB.Close() } + +// Reopen closes the database and reopens it. +func (db *DB) Reopen() error { + if err := db.DB.Close(); err != nil { + return err + } + + path, name := db.Path(), db.Name() + db.DB = pilosa.NewDB(path, name) + + if err := db.Open(); err != nil { + return err + } + return nil +} + +// MustSetBit sets a bit on the database. Panic on error. +func (db *DB) MustSetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool) { + changed, err := db.SetBit(name, bitmapID, profileID, t) + if err != nil { + panic(err) + } + return changed +} diff --git a/executor.go b/executor.go index 3ef806940..3a8152ff5 100644 --- a/executor.go +++ b/executor.go @@ -303,11 +303,28 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Rang frame = DefaultFrame } - f := e.Index.Fragment(db, frame, slice) + // Retrieve base frame. + f := e.Index.Frame(db, frame) if f == nil { - return NewBitmap(), nil + return &Bitmap{}, nil } - return f.Range(c.ID, c.StartTime, c.EndTime), nil + + // If no quantum exists then return an empty bitmap. + q := f.TimeQuantum() + if q == "" { + return &Bitmap{}, nil + } + + // Union bitmaps across all time-based subframes. + bm := &Bitmap{} + for _, subframe := range FramesByTimeRange(frame, c.StartTime, c.EndTime, q) { + f := e.Index.Fragment(db, subframe, slice) + if f == nil { + continue + } + bm = bm.Union(f.Bitmap(c.ID)) + } + return bm, nil } // executeUnionSlice executes a union() call for a local slice. @@ -404,15 +421,14 @@ func (e *Executor) executeSetBit(ctx context.Context, db string, c *pql.SetBit, for _, node := range e.Cluster.FragmentNodes(db, slice) { // Update locally if host matches. if node.Host == e.Host { - f, err := e.Index.CreateFragmentIfNotExists(db, c.Frame, slice) + db, err := e.Index.CreateDBIfNotExists(db) if err != nil { - return false, fmt.Errorf("fragment: %s", err) + return false, fmt.Errorf("db: %s", err) } - val, err := f.SetBit(c.ID, c.ProfileID, opt.Timestamp, opt.Quantum) + val, err := db.SetBit(c.Frame, c.ID, c.ProfileID, opt.Timestamp) if err != nil { return false, err - } - if val { + } else if val { ret = true } continue @@ -578,14 +594,13 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query, slices []uint64, opt *ExecOptions) (results []interface{}, err error) { // Encode request object. pbreq := &internal.QueryRequest{ - DB: proto.String(db), - Query: proto.String(q.String()), - Slices: slices, - Quantum: proto.Uint32(uint32(opt.Quantum)), - Remote: proto.Bool(true), + DB: db, + Query: q.String(), + Slices: slices, + Remote: true, } if opt.Timestamp != nil { - pbreq.Timestamp = proto.Int64(opt.Timestamp.UnixNano()) + pbreq.Timestamp = opt.Timestamp.UnixNano() } buf, err := proto.Marshal(pbreq) if err != nil { @@ -631,7 +646,7 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query } // Return an error, if specified on response. - if err := decodeError(pb.GetErr()); err != nil { + if err := decodeError(pb.Err); err != nil { return nil, err } @@ -647,11 +662,11 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query case *pql.TopN: v, err = decodePairs(pb.Results[i].GetPairs()), nil case *pql.Count: - v, err = pb.Results[i].GetN(), nil + v, err = pb.Results[i].N, nil case *pql.SetBit: - v, err = pb.Results[i].GetChanged(), nil + v, err = pb.Results[i].Changed, nil case *pql.ClearBit: - v, err = pb.Results[i].GetChanged(), nil + v, err = pb.Results[i].Changed, nil case *pql.SetBitmapAttrs: case *pql.SetProfileAttrs: default: @@ -836,7 +851,6 @@ type mapResponse struct { // ExecOptions represents an execution context for a single Execute() call. type ExecOptions struct { Timestamp *time.Time - Quantum TimeQuantum Remote bool } diff --git a/executor_test.go b/executor_test.go index fc53f6c0a..164d35acf 100644 --- a/executor_test.go +++ b/executor_test.go @@ -171,15 +171,15 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 5).SetBit(0, (5*SliceWidth)+100) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(10, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", 0).SetBit(0, 0) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -197,12 +197,12 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 2, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth+2, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 2) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth+2) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(1, SliceWidth) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -221,19 +221,19 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { defer idx.Close() // Set bits for bitmaps 0, 10, & 20 across two slices. - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 0) + idx.MustCreateFragmentIfNotExists("d", "f", 0).SetBit(0, 1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(0, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(10, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "f", 1).SetBit(20, SliceWidth+2) // Create an intersecting bitmap. - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1, nil, 0) - idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2, nil, 0) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+1) + idx.MustCreateFragmentIfNotExists("d", "other", 1).SetBit(100, SliceWidth+2) // Execute query. e := NewExecutor(idx.Index, NewCluster(1)) @@ -253,15 +253,23 @@ func TestExecutor_Execute_Range(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - f := idx.MustCreateFragmentIfNotExists("d", "f.t", 0) - if _, err := f.SetBit(1, 100, MustParseTime("2000-01-01 00:00"), pilosa.YMD); err != nil { - t.Fatal(err) - } + db := idx.MustCreateDBIfNotExists("d") + db.SetTimeQuantum(pilosa.TimeQuantum("YMDH")) + db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-31 00:00")) + db.MustSetBit("f", 1, 3, MustParseTimePtr("2000-01-01 00:00")) + db.MustSetBit("f", 1, 4, MustParseTimePtr("2000-01-02 00:00")) + db.MustSetBit("f", 1, 5, MustParseTimePtr("2000-02-01 00:00")) + db.MustSetBit("f", 1, 6, MustParseTimePtr("2001-01-01 00:00")) + db.MustSetBit("f", 1, 7, MustParseTimePtr("2002-01-01 02:00")) + + db.MustSetBit("f", 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early + db.MustSetBit("f", 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late + db.MustSetBit("f", 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap e := NewExecutor(idx.Index, NewCluster(1)) - if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f.t, start="2000-01-01T00:00", end="2000-01-01T01:00")`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{100}) { + } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { t.Fatalf("unexpected bits: %+v", bits) } } diff --git a/fragment.go b/fragment.go index d0bb2e73e..3eea7cf03 100644 --- a/fragment.go +++ b/fragment.go @@ -238,7 +238,7 @@ func (f *Fragment) openCache() error { // Read in all bitmaps by ID. // This will cause them to be added to the cache. - for _, bitmapID := range pb.GetBitmapIDs() { + for _, bitmapID := range pb.BitmapIDs { n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) f.cache.Add(bitmapID, n) } @@ -323,11 +323,6 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { } bm.InvalidateCount() - // f.storage.ForEachRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth, func(i uint64) { - // profileID := (f.slice * SliceWidth) + (i % SliceWidth) - // bm.SetBit(profileID) - // }) - // Update cache. f.cache.Add(bitmapID, bm.Count()) @@ -336,15 +331,9 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { // SetBit sets a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(bitmapID, profileID uint64, t *time.Time, q TimeQuantum) (changed bool, err error) { +func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - - // Set time bits if this is a time-frame and a timestamp is specified. - if strings.HasSuffix(f.frame, FrameSuffixTime) && t != nil { - return f.setTimeBit(bitmapID, profileID, *t, q) - } - return f.setBit(bitmapID, profileID) } @@ -379,17 +368,6 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, bool error) return changed, nil } -func (f *Fragment) setTimeBit(bitmapID, profileID uint64, t time.Time, q TimeQuantum) (changed bool, err error) { - for _, timeID := range TimeIDsFromQuantum(q, t, bitmapID) { - if v, err := f.setBit(timeID, profileID); err != nil { - return changed, fmt.Errorf("set time bit: t=%s, q=%s, err=%s", t, q, err) - } else if v { - changed = true - } - } - return changed, nil -} - // ClearBit clears a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) { @@ -601,24 +579,6 @@ type TopOptions struct { FilterValues []interface{} } -func (f *Fragment) Range(bitmapID uint64, start, end time.Time) *Bitmap { - f.mu.Lock() - defer f.mu.Unlock() - - // Retrieve a list of bitmap ids for a given time range. - bitmapIDs := TimeIDsFromRange(start, end, bitmapID) - if len(bitmapIDs) == 0 { - return NewBitmap() - } - - // Union all bitmap ids from the time range. - bm := f.bitmap(bitmapIDs[0]) - for _, id := range bitmapIDs[1:] { - bm = bm.Union(f.bitmap(id)) - } - return bm -} - // Checksum returns a checksum for the entire fragment. // If two fragments have the same checksum then they have the same data. func (f *Fragment) Checksum() []byte { diff --git a/fragment_test.go b/fragment_test.go index 11ce52dca..da2edac6c 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -27,11 +27,11 @@ func TestFragment_SetBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(120, 1, nil, 0); err != nil { + if _, err := f.SetBit(120, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 6, nil, 0); err != nil { + } else if _, err := f.SetBit(120, 6); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(121, 0, nil, 0); err != nil { + } else if _, err := f.SetBit(121, 0); err != nil { t.Fatal(err) } @@ -58,9 +58,9 @@ func TestFragment_ClearBit(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1, nil, 0); err != nil { + if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { + } else if _, err := f.SetBit(1000, 2); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -85,9 +85,9 @@ func TestFragment_Snapshot(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if _, err := f.SetBit(1000, 1, nil, 0); err != nil { + if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(1000, 2, nil, 0); err != nil { + } else if _, err := f.SetBit(1000, 2); err != nil { t.Fatal(err) } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -114,11 +114,11 @@ func TestFragment_ForEachBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if _, err := f.SetBit(100, 20, nil, 0); err != nil { + if _, err := f.SetBit(100, 20); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 38, nil, 0); err != nil { + } else if _, err := f.SetBit(2, 38); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 37, nil, 0); err != nil { + } else if _, err := f.SetBit(2, 37); err != nil { t.Fatal(err) } @@ -284,9 +284,9 @@ func TestFragment_Checksum(t *testing.T) { // Retrieve checksum and set bits. orig := f.Checksum() - if _, err := f.SetBit(1, 200, nil, 0); err != nil { + if _, err := f.SetBit(1, 200); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.HashBlockSize*2, 200, nil, 0); err != nil { + } else if _, err := f.SetBit(pilosa.HashBlockSize*2, 200); err != nil { t.Fatal(err) } @@ -305,7 +305,7 @@ func TestFragment_Blocks(t *testing.T) { var prev []pilosa.FragmentBlock // Set first bit. - if _, err := f.SetBit(0, 0, nil, 0); err != nil { + if _, err := f.SetBit(0, 0); err != nil { t.Fatal(err) } blocks := f.Blocks() @@ -315,7 +315,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different bitmap. - if _, err := f.SetBit(20, 0, nil, 0); err != nil { + if _, err := f.SetBit(20, 0); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -325,7 +325,7 @@ func TestFragment_Blocks(t *testing.T) { prev = blocks // Set bit on different profile. - if _, err := f.SetBit(20, 100, nil, 0); err != nil { + if _, err := f.SetBit(20, 100); err != nil { t.Fatal(err) } blocks = f.Blocks() @@ -340,7 +340,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { defer f.Close() // Set bits on a different block. - if _, err := f.SetBit(100, 1, nil, 0); err != nil { + if _, err := f.SetBit(100, 1); err != nil { t.Fatal(err) } @@ -359,7 +359,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0, nil, 0); err != nil { + if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) } } @@ -391,7 +391,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if _, err := f.SetBit(i, 0, nil, 0); err != nil { + if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) } } @@ -422,9 +422,9 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { defer f0.Close() // Set and then clear bits on the fragment. - if _, err := f0.SetBit(1000, 1, nil, 0); err != nil { + if _, err := f0.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if _, err := f0.SetBit(1000, 2, nil, 0); err != nil { + } else if _, err := f0.SetBit(1000, 2); err != nil { t.Fatal(err) } else if _, err := f0.ClearBit(1000, 1); err != nil { t.Fatal(err) @@ -527,12 +527,12 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Generate some intersecting data. for i := 0; i < 10000; i += 2 { - if _, err := f.SetBit(1, uint64(i), nil, 0); err != nil { + if _, err := f.SetBit(1, uint64(i)); err != nil { b.Fatal(err) } } for i := 0; i < 10000; i += 3 { - if _, err := f.SetBit(2, uint64(i), nil, 0); err != nil { + if _, err := f.SetBit(2, uint64(i)); err != nil { b.Fatal(err) } } @@ -609,7 +609,7 @@ func (f *Fragment) Reopen() error { // This function does not accept a timestamp or quantum. func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { for _, profileID := range profileIDs { - if _, err := f.SetBit(bitmapID, profileID, nil, 0); err != nil { + if _, err := f.SetBit(bitmapID, profileID); err != nil { panic(err) } } diff --git a/frame.go b/frame.go index c8f18f8a1..b215d80d2 100644 --- a/frame.go +++ b/frame.go @@ -2,26 +2,28 @@ package pilosa import ( "fmt" + "io/ioutil" "os" "path/filepath" "strconv" "sync" + + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/internal" ) const ( - // FrameSuffixTime is the suffix used for time-based frames. - FrameSuffixTime = ".t" - // FrameSuffixRank is the suffix used for rank-based frames. FrameSuffixRank = ".n" ) // Frame represents a container for fragments. type Frame struct { - mu sync.Mutex - path string - db string - name string + mu sync.Mutex + path string + db string + name string + timeQuantum TimeQuantum // Fragments by slice. fragments map[uint64]*Fragment @@ -80,6 +82,10 @@ func (f *Frame) Open() error { return err } + if err := f.loadMeta(); err != nil { + return err + } + if err := f.openFragments(); err != nil { return err } @@ -134,6 +140,45 @@ func (f *Frame) openFragments() error { return nil } +// loadMeta reads meta data for the frame, if any. +func (f *Frame) loadMeta() error { + var pb internal.Frame + + // Read data from meta file. + buf, err := ioutil.ReadFile(filepath.Join(f.path, "meta")) + if os.IsNotExist(err) { + f.timeQuantum = "" + return nil + } else if err != nil { + return err + } else { + if err := proto.Unmarshal(buf, &pb); err != nil { + return err + } + } + + // Copy metadata fields. + f.timeQuantum = TimeQuantum(pb.TimeQuantum) + + return nil +} + +// saveMeta writes meta data for the frame. +func (f *Frame) saveMeta() error { + // Marshal metadata. + buf, err := proto.Marshal(&internal.Frame{TimeQuantum: string(f.timeQuantum)}) + if err != nil { + return err + } + + // Write to meta file. + if err := ioutil.WriteFile(filepath.Join(f.path, "meta"), buf, 0666); err != nil { + return err + } + + return nil +} + // Close closes the frame and its fragments. func (f *Frame) Close() error { f.mu.Lock() @@ -153,6 +198,34 @@ func (f *Frame) Close() error { return nil } +// TimeQuantum returns the time quantum for the frame. +func (f *Frame) TimeQuantum() TimeQuantum { + f.mu.Lock() + defer f.mu.Unlock() + return f.timeQuantum +} + +// SetTimeQuantum sets the time quantum for the frame. +func (f *Frame) SetTimeQuantum(q TimeQuantum) error { + f.mu.Lock() + defer f.mu.Unlock() + + // Validate input. + if !q.Valid() { + return ErrInvalidTimeQuantum + } + + // Update value on frame. + f.timeQuantum = q + + // Perist meta data to disk. + if err := f.saveMeta(); err != nil { + return err + } + + return nil +} + // FragmentPath returns the path to a fragment in the frame. func (f *Frame) FragmentPath(slice uint64) string { return filepath.Join(f.path, strconv.FormatUint(slice, 10)) @@ -213,6 +286,16 @@ func (f *Frame) newFragment(path string, slice uint64) *Fragment { return frag } +// SetBit sets a bit within the frame. +func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) { + slice := bitmapID / SliceWidth + frag, err := f.CreateFragmentIfNotExists(slice) + if err != nil { + return changed, err + } + return frag.SetBit(bitmapID, profileID) +} + type frameSlice []*Frame func (p frameSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/frame_test.go b/frame_test.go index 01d508ef9..186db0c62 100644 --- a/frame_test.go +++ b/frame_test.go @@ -34,6 +34,26 @@ func TestFrame_CreateFragmentIfNotExists(t *testing.T) { } } +// Ensure frame can set its time quantum. +func TestFrame_SetTimeQuantum(t *testing.T) { + f := MustOpenFrame() + defer f.Close() + + // Set & retrieve time quantum. + if err := f.SetTimeQuantum(pilosa.TimeQuantum("YMDH")); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %s", q) + } + + // Reload frame and verify that it is persisted. + if err := f.Reopen(); err != nil { + t.Fatal(err) + } else if q := f.TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum (reopen): %s", q) + } +} + // Frame represents a test wrapper for pilosa.Frame. type Frame struct { *pilosa.Frame @@ -63,3 +83,18 @@ func (f *Frame) Close() error { defer os.RemoveAll(f.Path()) return f.Frame.Close() } + +// Reopen closes the database and reopens it. +func (f *Frame) Reopen() error { + if err := f.Frame.Close(); err != nil { + return err + } + + path, db, name := f.Path(), f.DB(), f.Name() + f.Frame = pilosa.NewFrame(path, db, name) + + if err := f.Open(); err != nil { + return err + } + return nil +} diff --git a/handler.go b/handler.go index c278ddbd7..1445d8c7c 100644 --- a/handler.go +++ b/handler.go @@ -114,6 +114,13 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } + case "/db/time_quantum": + switch r.Method { + case "PATCH": + h.handlePatchDBTimeQuantum(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } case "/db/attr/diff": switch r.Method { case "POST": @@ -121,6 +128,20 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } + case "/frame": + switch r.Method { + case "DELETE": + h.handleDeleteFrame(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } + case "/frame/time_quantum": + switch r.Method { + case "PATCH": + h.handlePatchFrameTimeQuantum(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } case "/frame/attr/diff": switch r.Method { case "POST": @@ -203,7 +224,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { // Build execution options. opt := &ExecOptions{ Timestamp: req.Timestamp, - Quantum: req.Quantum, Remote: req.Remote, } @@ -301,6 +321,48 @@ type deleteDBRequest struct { type deleteDBResponse struct{} +// handlePatchDBTimeQuantum handles PATCH /db/time_quantum request. +func (h *Handler) handlePatchDBTimeQuantum(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req patchDBTimeQuantumRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Validate quantum. + tq, err := ParseTimeQuantum(req.TimeQuantum) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Retrieve database by name. + db, err := h.Index.CreateDBIfNotExists(req.DB) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Set default time quantum on database. + if err := db.SetTimeQuantum(tq); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(patchDBTimeQuantumResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type patchDBTimeQuantumRequest struct { + DB string `json:"db"` + TimeQuantum string `json:"time_quantum"` +} + +type patchDBTimeQuantumResponse struct{} + // handlePostDBAttrDiff handles POST /db/attr/diff requests. func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -357,6 +419,86 @@ type postDBAttrDiffResponse struct { Attrs map[uint64]map[string]interface{} `json:"attrs"` } +// handleDeleteFrame handles DELETE /frame request. +func (h *Handler) handleDeleteFrame(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req deleteFrameRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Find database. + db := h.Index.DB(req.DB) + if db == nil { + if err := json.NewEncoder(w).Encode(deleteDBResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } + return + } + + // Delete frame from the database. + if err := db.DeleteFrame(req.Frame); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(deleteFrameResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type deleteFrameRequest struct { + DB string `json:"db"` + Frame string `json:"frame"` +} + +type deleteFrameResponse struct{} + +// handlePatchFrameTimeQuantum handles PATCH /frame/time_quantum request. +func (h *Handler) handlePatchFrameTimeQuantum(w http.ResponseWriter, r *http.Request) { + // Decode request. + var req patchFrameTimeQuantumRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Validate quantum. + tq, err := ParseTimeQuantum(req.TimeQuantum) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + // Retrieve database by name. + f, err := h.Index.CreateFrameIfNotExists(req.DB, req.Frame) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Set default time quantum on database. + if err := f.SetTimeQuantum(tq); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + + // Encode response. + if err := json.NewEncoder(w).Encode(patchFrameTimeQuantumResponse{}); err != nil { + h.logger().Printf("response encoding error: %s", err) + } +} + +type patchFrameTimeQuantumRequest struct { + DB string `json:"db"` + Frame string `json:"frame"` + TimeQuantum string `json:"time_quantum"` +} + +type patchFrameTimeQuantumResponse struct{} + // handlePostFrameAttrDiff handles POST /frame/attr/diff requests. func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request) { // Decode request. @@ -497,7 +639,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { } // Parse time granularity. - quantum := YMDH + quantum := TimeQuantum("YMDH") if s := q.Get("time_granularity"); s != "" { v, err := ParseTimeQuantum(s) if err != nil { @@ -563,7 +705,7 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - db, frame, slice := req.GetDB(), req.GetFrame(), req.GetSlice() + db, frame, slice := req.DB, req.Frame, req.Slice // Validate that this handler owns the slice. if !h.Cluster.OwnsFragment(h.Host, db, slice) { @@ -580,16 +722,16 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { http.Error(w, "fragment error", http.StatusInternalServerError) return } - h.logger().Println("Import into Fragment:", db, frame, slice, len(req.GetProfileIDs())) + h.logger().Println("Import into Fragment:", db, frame, slice, len(req.ProfileIDs)) // Import into fragment. - err = f.Import(req.GetBitmapIDs(), req.GetProfileIDs()) + err = f.Import(req.BitmapIDs, req.ProfileIDs) if err != nil { - h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", db, frame, slice, len(req.GetProfileIDs()), err) + h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", db, frame, slice, len(req.ProfileIDs), err) } // Marshal response object. - buf, e := proto.Marshal(&internal.ImportResponse{Err: proto.String(errorString(err))}) + buf, e := proto.Marshal(&internal.ImportResponse{Err: errorString(err)}) if e != nil { http.Error(w, fmt.Sprintf("marshal import response: %s", err), http.StatusInternalServerError) return @@ -735,7 +877,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ } // Retrieve fragment from index. - f := h.Index.Fragment(req.GetDB(), req.GetFrame(), req.GetSlice()) + f := h.Index.Fragment(req.DB, req.Frame, req.Slice) if f == nil { http.Error(w, ErrFragmentNotFound.Error(), http.StatusNotFound) return @@ -744,7 +886,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ // Read data var resp internal.BlockDataResponse if f != nil { - resp.BitmapIDs, resp.ProfileIDs = f.BlockData(int(req.GetBlock())) + resp.BitmapIDs, resp.ProfileIDs = f.BlockData(int(req.Block)) } // Encode response. @@ -922,16 +1064,16 @@ type QueryRequest struct { func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { req := &QueryRequest{ - DB: pb.GetDB(), - Query: pb.GetQuery(), - Slices: pb.GetSlices(), - Profiles: pb.GetProfiles(), - Quantum: TimeQuantum(pb.GetQuantum()), - Remote: pb.GetRemote(), + DB: pb.DB, + Query: pb.Query, + Slices: pb.Slices, + Profiles: pb.Profiles, + Quantum: TimeQuantum(pb.Quantum), + Remote: pb.Remote, } - if pb.Timestamp != nil { - t := time.Unix(0, pb.GetTimestamp()) + if pb.Timestamp != 0 { + t := time.Unix(0, pb.Timestamp) req.Timestamp = &t } @@ -981,14 +1123,14 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { case []Pair: pb.Results[i].Pairs = encodePairs(result) case uint64: - pb.Results[i].N = proto.Uint64(result) + pb.Results[i].N = result case bool: - pb.Results[i].Changed = proto.Bool(result) + pb.Results[i].Changed = result } } if resp.Err != nil { - pb.Err = proto.String(resp.Err.Error()) + pb.Err = resp.Err.Error() } return pb diff --git a/handler_test.go b/handler_test.go index df157a605..6e998191a 100644 --- a/handler_test.go +++ b/handler_test.go @@ -117,8 +117,8 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) { // Generate request body. reqBody, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String("db0"), - Query: proto.String("Count(Bitmap(100))"), + DB: "db0", + Query: "Count(Bitmap(100))", Slices: []uint64{0, 1}, }) if err != nil { @@ -181,7 +181,7 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if n := resp.Results[0].GetN(); n != 100 { + } else if n := resp.Results[0].N; n != 100 { t.Fatalf("unexpected n: %d", n) } } @@ -256,15 +256,15 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { + } else if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) - } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { + } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].GetKey(), attrs[1].GetUintValue(); k != "c" || v != uint64(1) { + } else if k, v := attrs[1].Key, attrs[1].UintValue; k != "c" || v != uint64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].GetKey(), attrs[2].GetBoolValue(); k != "d" || v != true { + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v) } } @@ -292,9 +292,9 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Encode request body. buf, err := proto.Marshal(&internal.QueryRequest{ - DB: proto.String("d"), - Query: proto.String("Bitmap(100)"), - Profiles: proto.Bool(true), + DB: "d", + Query: "Bitmap(100)", + Profiles: true, }) if err != nil { t.Fatal(err) @@ -313,25 +313,25 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if bits := resp.Results[0].GetBitmap().GetBits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { + if bits := resp.Results[0].Bitmap.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { t.Fatalf("unexpected bits: %+v", bits) - } else if attrs := resp.Results[0].GetBitmap().GetAttrs(); len(attrs) != 3 { + } else if attrs := resp.Results[0].Bitmap.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) - } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { + } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].GetKey(), attrs[1].GetUintValue(); k != "c" || v != uint64(1) { + } else if k, v := attrs[1].Key, attrs[1].UintValue; k != "c" || v != uint64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) - } else if k, v := attrs[2].GetKey(), attrs[2].GetBoolValue(); k != "d" || v != true { + } else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v) } - if a := resp.GetProfiles(); len(a) != 1 { + if a := resp.Profiles; len(a) != 1 { t.Fatalf("unexpected profiles length: %d", len(a)) - } else if a[0].GetID() != 1 { - t.Fatalf("unexpected id: %d", a[0].GetID()) - } else if len(a[0].GetAttrs()) != 1 { + } else if a[0].ID != 1 { + t.Fatalf("unexpected id: %d", a[0].ID) + } else if len(a[0].Attrs) != 1 { t.Fatalf("unexpected profile attr length: %d", len(a)) - } else if k, v := a[0].GetAttrs()[0].GetKey(), a[0].GetAttrs()[0].GetStringValue(); k != "x" || v != "y" { + } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) } } @@ -415,7 +415,7 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) { var resp internal.QueryResponse if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) - } else if s := resp.GetErr(); s != `marker` { + } else if s := resp.Err; s != `marker` { t.Fatalf("unexpected error: %s", s) } } @@ -477,6 +477,63 @@ func TestHandler_DB_Delete(t *testing.T) { } } +// Ensure handler can delete a frame. +func TestHandler_DeleteFrame(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + if _, err := idx.CreateFrameIfNotExists("d0", "f1"); err != nil { + t.Fatal(err) + } + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/frame", strings.NewReader(`{"db":"d0","frame":"f1"}`))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if f := idx.DB("d0").Frame("f1"); f != nil { + t.Fatal("expected nil frame") + } +} + +// Ensure handler can set the DB time quantum. +func TestHandler_SetDBTimeQuantum(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/db/time_quantum", strings.NewReader(`{"db":"d0","time_quantum":"ymdh"}`))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if q := idx.DB("d0").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected time quantum: %s", q) + } +} + +// Ensure handler can set the frame time quantum. +func TestHandler_SetFrameTimeQuantum(t *testing.T) { + idx := MustOpenIndex() + defer idx.Close() + + h := NewHandler() + h.Index = idx.Index + w := httptest.NewRecorder() + h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/frame/time_quantum", strings.NewReader(`{"db":"d0","frame":"f1","time_quantum":"ymdh"}`))) + if w.Code != http.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } else if body := w.Body.String(); body != `{}`+"\n" { + t.Fatalf("unexpected body: %s", body) + } else if q := idx.DB("d0").Frame("f1").TimeQuantum(); q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected time quantum: %s", q) + } +} + // Ensure the handler can return data in differing blocks for a database. func TestHandler_DB_AttrStore_Diff(t *testing.T) { idx := MustOpenIndex() diff --git a/index_test.go b/index_test.go index 7392adb5f..4707139d7 100644 --- a/index_test.go +++ b/index_test.go @@ -19,11 +19,11 @@ func TestIndex_DeleteDB(t *testing.T) { // Write bits to separate databases. f0 := idx.MustCreateFragmentIfNotExists("d0", "f", 0) - if _, err := f0.SetBit(100, 200, nil, 0); err != nil { + if _, err := f0.SetBit(100, 200); err != nil { t.Fatal(err) } f1 := idx.MustCreateFragmentIfNotExists("d1", "f", 0) - if _, err := f1.SetBit(100, 200, nil, 0); err != nil { + if _, err := f1.SetBit(100, 200); err != nil { t.Fatal(err) } @@ -74,18 +74,18 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Set data on the local index. f := idx0.MustCreateFragmentIfNotExists("d", "f", 0) - if _, err := f.SetBit(0, 10, nil, 0); err != nil { + if _, err := f.SetBit(0, 10); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(2, 20, nil, 0); err != nil { + } else if _, err := f.SetBit(2, 20); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 10, nil, 0); err != nil { + } else if _, err := f.SetBit(120, 10); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(200, 4, nil, 0); err != nil { + } else if _, err := f.SetBit(200, 4); err != nil { t.Fatal(err) } f = idx0.MustCreateFragmentIfNotExists("d", "f0", 1) - if _, err := f.SetBit(9, SliceWidth+5, nil, 0); err != nil { + if _, err := f.SetBit(9, SliceWidth+5); err != nil { t.Fatal(err) } @@ -93,20 +93,20 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Set data on the remote index. f = idx1.MustCreateFragmentIfNotExists("d", "f", 0) - if _, err := f.SetBit(0, 4000, nil, 0); err != nil { + if _, err := f.SetBit(0, 4000); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(3, 10, nil, 0); err != nil { + } else if _, err := f.SetBit(3, 10); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(120, 10, nil, 0); err != nil { + } else if _, err := f.SetBit(120, 10); err != nil { t.Fatal(err) } f = idx1.MustCreateFragmentIfNotExists("y", "z", 3) - if _, err := f.SetBit(10, (3*SliceWidth)+4, nil, 0); err != nil { + if _, err := f.SetBit(10, (3*SliceWidth)+4); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+5, nil, 0); err != nil { + } else if _, err := f.SetBit(10, (3*SliceWidth)+5); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(10, (3*SliceWidth)+7, nil, 0); err != nil { + } else if _, err := f.SetBit(10, (3*SliceWidth)+7); err != nil { t.Fatal(err) } @@ -190,6 +190,24 @@ func (i *Index) Close() error { return i.Index.Close() } +// MustCreateDBIfNotExists returns a given db. Panic on error. +func (i *Index) MustCreateDBIfNotExists(db string) *DB { + d, err := i.Index.CreateDBIfNotExists(db) + if err != nil { + panic(err) + } + return &DB{DB: d} +} + +// MustCreateFrameIfNotExists returns a given frame. Panic on error. +func (i *Index) MustCreateFrameIfNotExists(db, frame string) *Frame { + f, err := i.Index.CreateFrameIfNotExists(db, frame) + if err != nil { + panic(err) + } + return &Frame{Frame: f} +} + // MustCreateFragmentIfNotExists returns a given fragment. Panic on error. func (i *Index) MustCreateFragmentIfNotExists(db, frame string, slice uint64) *Fragment { f, err := i.Index.CreateFragmentIfNotExists(db, frame, slice) diff --git a/internal/internal.go b/internal/internal.go index 574c0ee5c..3d322d82a 100644 --- a/internal/internal.go +++ b/internal/internal.go @@ -7,6 +7,8 @@ import ( "github.com/gogo/protobuf/proto" ) +//go:generate protoc --gofast_out=. internal.proto + type Request proto.Message type Response proto.Message diff --git a/internal/internal.pb.go b/internal/internal.pb.go index 0adcfb4ca..f4ea8a9e8 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -1,36 +1,40 @@ // Code generated by protoc-gen-gogo. -// source: internal/internal.proto +// source: internal.proto // DO NOT EDIT! /* -Package internal is a generated protocol buffer package. + Package internal is a generated protocol buffer package. -It is generated from these files: - internal/internal.proto + It is generated from these files: + internal.proto -It has these top-level messages: - Bitmap - Pair - Bit - Profile - Attr - AttrMap - QueryRequest - QueryResponse - QueryResult - ImportRequest - ImportResponse - BlockDataRequest - BlockDataResponse - Cache - MaxSlicesResponse + It has these top-level messages: + DB + Frame + Bitmap + Pair + Bit + Profile + Attr + AttrMap + QueryRequest + QueryResponse + QueryResult + ImportRequest + ImportResponse + BlockDataRequest + BlockDataResponse + Cache + MaxSlicesResponse */ package internal -import proto "github.com/gogo/protobuf/proto" +import proto "github.com/golang/protobuf/proto" import fmt "fmt" import math "math" +import io "io" + // Reference imports to suppress errors if they are not otherwise used. var _ = proto.Marshal var _ = fmt.Errorf @@ -40,25 +44,35 @@ var _ = math.Inf // is compatible with the proto package it is being compiled against. // A compilation error at this line likely means your copy of the // proto package needs to be updated. -const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type DB struct { + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` +} + +func (m *DB) Reset() { *m = DB{} } +func (m *DB) String() string { return proto.CompactTextString(m) } +func (*DB) ProtoMessage() {} +func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } + +type Frame struct { + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` +} + +func (m *Frame) Reset() { *m = Frame{} } +func (m *Frame) String() string { return proto.CompactTextString(m) } +func (*Frame) ProtoMessage() {} +func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } type Bitmap struct { - Bits []uint64 `protobuf:"varint,1,rep,name=Bits,json=bits" json:"Bits,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` - XXX_unrecognized []byte `json:"-"` + Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Bitmap) Reset() { *m = Bitmap{} } func (m *Bitmap) String() string { return proto.CompactTextString(m) } func (*Bitmap) ProtoMessage() {} -func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } - -func (m *Bitmap) GetBits() []uint64 { - if m != nil { - return m.Bits - } - return nil -} +func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} } func (m *Bitmap) GetAttrs() []*Attr { if m != nil { @@ -68,72 +82,34 @@ func (m *Bitmap) GetAttrs() []*Attr { } type Pair struct { - Key *uint64 `protobuf:"varint,1,req,name=Key,json=key" json:"Key,omitempty"` - Count *uint64 `protobuf:"varint,2,req,name=Count,json=count" json:"Count,omitempty"` - XXX_unrecognized []byte `json:"-"` + Key uint64 `protobuf:"varint,1,opt,name=Key,proto3" json:"Key,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } func (m *Pair) Reset() { *m = Pair{} } func (m *Pair) String() string { return proto.CompactTextString(m) } func (*Pair) ProtoMessage() {} -func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } - -func (m *Pair) GetKey() uint64 { - if m != nil && m.Key != nil { - return *m.Key - } - return 0 -} - -func (m *Pair) GetCount() uint64 { - if m != nil && m.Count != nil { - return *m.Count - } - return 0 -} +func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } type Bit struct { - BitmapID *uint64 `protobuf:"varint,1,req,name=BitmapID,json=bitmapID" json:"BitmapID,omitempty"` - ProfileID *uint64 `protobuf:"varint,2,req,name=ProfileID,json=profileID" json:"ProfileID,omitempty"` - XXX_unrecognized []byte `json:"-"` + BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,proto3" json:"BitmapID,omitempty"` + ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,proto3" json:"ProfileID,omitempty"` } func (m *Bit) Reset() { *m = Bit{} } func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} -func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{2} } - -func (m *Bit) GetBitmapID() uint64 { - if m != nil && m.BitmapID != nil { - return *m.BitmapID - } - return 0 -} - -func (m *Bit) GetProfileID() uint64 { - if m != nil && m.ProfileID != nil { - return *m.ProfileID - } - return 0 -} +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } type Profile struct { - ID *uint64 `protobuf:"varint,1,req,name=ID,json=iD" json:"ID,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` - XXX_unrecognized []byte `json:"-"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Profile) Reset() { *m = Profile{} } func (m *Profile) String() string { return proto.CompactTextString(m) } func (*Profile) ProtoMessage() {} -func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } - -func (m *Profile) GetID() uint64 { - if m != nil && m.ID != nil { - return *m.ID - } - return 0 -} +func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} } func (m *Profile) GetAttrs() []*Attr { if m != nil { @@ -143,55 +119,26 @@ func (m *Profile) GetAttrs() []*Attr { } type Attr struct { - Key *string `protobuf:"bytes,1,req,name=Key,json=key" json:"Key,omitempty"` - StringValue *string `protobuf:"bytes,2,opt,name=StringValue,json=stringValue" json:"StringValue,omitempty"` - UintValue *uint64 `protobuf:"varint,3,opt,name=UintValue,json=uintValue" json:"UintValue,omitempty"` - BoolValue *bool `protobuf:"varint,4,opt,name=BoolValue,json=boolValue" json:"BoolValue,omitempty"` - XXX_unrecognized []byte `json:"-"` + Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` + Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` + StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` + UintValue uint64 `protobuf:"varint,4,opt,name=UintValue,proto3" json:"UintValue,omitempty"` + BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` } func (m *Attr) Reset() { *m = Attr{} } func (m *Attr) String() string { return proto.CompactTextString(m) } func (*Attr) ProtoMessage() {} -func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } - -func (m *Attr) GetKey() string { - if m != nil && m.Key != nil { - return *m.Key - } - return "" -} - -func (m *Attr) GetStringValue() string { - if m != nil && m.StringValue != nil { - return *m.StringValue - } - return "" -} - -func (m *Attr) GetUintValue() uint64 { - if m != nil && m.UintValue != nil { - return *m.UintValue - } - return 0 -} - -func (m *Attr) GetBoolValue() bool { - if m != nil && m.BoolValue != nil { - return *m.BoolValue - } - return false -} +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` - XXX_unrecognized []byte `json:"-"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *AttrMap) Reset() { *m = AttrMap{} } func (m *AttrMap) String() string { return proto.CompactTextString(m) } func (*AttrMap) ProtoMessage() {} -func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{5} } +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -201,88 +148,30 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB,json=dB" json:"DB,omitempty"` - Query *string `protobuf:"bytes,2,req,name=Query,json=query" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,3,rep,name=Slices,json=slices" json:"Slices,omitempty"` - Profiles *bool `protobuf:"varint,4,opt,name=Profiles,json=profiles" json:"Profiles,omitempty"` - Timestamp *int64 `protobuf:"varint,5,opt,name=Timestamp,json=timestamp" json:"Timestamp,omitempty"` - Quantum *uint32 `protobuf:"varint,6,opt,name=Quantum,json=quantum" json:"Quantum,omitempty"` - Remote *bool `protobuf:"varint,7,opt,name=Remote,json=remote" json:"Remote,omitempty"` - XXX_unrecognized []byte `json:"-"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Query string `protobuf:"bytes,2,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,3,rep,packed,name=Slices" json:"Slices,omitempty"` + Profiles bool `protobuf:"varint,4,opt,name=Profiles,proto3" json:"Profiles,omitempty"` + Timestamp int64 `protobuf:"varint,5,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` + Quantum string `protobuf:"bytes,6,opt,name=Quantum,proto3" json:"Quantum,omitempty"` + Remote bool `protobuf:"varint,7,opt,name=Remote,proto3" json:"Remote,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } func (m *QueryRequest) String() string { return proto.CompactTextString(m) } func (*QueryRequest) ProtoMessage() {} -func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } - -func (m *QueryRequest) GetDB() string { - if m != nil && m.DB != nil { - return *m.DB - } - return "" -} - -func (m *QueryRequest) GetQuery() string { - if m != nil && m.Query != nil { - return *m.Query - } - return "" -} - -func (m *QueryRequest) GetSlices() []uint64 { - if m != nil { - return m.Slices - } - return nil -} - -func (m *QueryRequest) GetProfiles() bool { - if m != nil && m.Profiles != nil { - return *m.Profiles - } - return false -} - -func (m *QueryRequest) GetTimestamp() int64 { - if m != nil && m.Timestamp != nil { - return *m.Timestamp - } - return 0 -} - -func (m *QueryRequest) GetQuantum() uint32 { - if m != nil && m.Quantum != nil { - return *m.Quantum - } - return 0 -} - -func (m *QueryRequest) GetRemote() bool { - if m != nil && m.Remote != nil { - return *m.Remote - } - return false -} +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } type QueryResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err,json=err" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results,json=results" json:"Results,omitempty"` - Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles,json=profiles" json:"Profiles,omitempty"` - XXX_unrecognized []byte `json:"-"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` } func (m *QueryResponse) Reset() { *m = QueryResponse{} } func (m *QueryResponse) String() string { return proto.CompactTextString(m) } func (*QueryResponse) ProtoMessage() {} -func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{7} } - -func (m *QueryResponse) GetErr() string { - if m != nil && m.Err != nil { - return *m.Err - } - return "" -} +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} } func (m *QueryResponse) GetResults() []*QueryResult { if m != nil { @@ -299,17 +188,16 @@ func (m *QueryResponse) GetProfiles() []*Profile { } type QueryResult struct { - Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap,json=bitmap" json:"Bitmap,omitempty"` - N *uint64 `protobuf:"varint,2,opt,name=N,json=n" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,json=pairs" json:"Pairs,omitempty"` - Changed *bool `protobuf:"varint,4,opt,name=Changed,json=changed" json:"Changed,omitempty"` - XXX_unrecognized []byte `json:"-"` + Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } func (m *QueryResult) String() string { return proto.CompactTextString(m) } func (*QueryResult) ProtoMessage() {} -func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } +func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} } func (m *QueryResult) GetBitmap() *Bitmap { if m != nil { @@ -318,13 +206,6 @@ func (m *QueryResult) GetBitmap() *Bitmap { return nil } -func (m *QueryResult) GetN() uint64 { - if m != nil && m.N != nil { - return *m.N - } - return 0 -} - func (m *QueryResult) GetPairs() []*Pair { if m != nil { return m.Pairs @@ -332,171 +213,67 @@ func (m *QueryResult) GetPairs() []*Pair { return nil } -func (m *QueryResult) GetChanged() bool { - if m != nil && m.Changed != nil { - return *m.Changed - } - return false -} - type ImportRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB,json=dB" json:"DB,omitempty"` - Frame *string `protobuf:"bytes,2,req,name=Frame,json=frame" json:"Frame,omitempty"` - Slice *uint64 `protobuf:"varint,3,req,name=Slice,json=slice" json:"Slice,omitempty"` - BitmapIDs []uint64 `protobuf:"varint,4,rep,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,5,rep,name=ProfileIDs,json=profileIDs" json:"ProfileIDs,omitempty"` - XXX_unrecognized []byte `json:"-"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` } func (m *ImportRequest) Reset() { *m = ImportRequest{} } func (m *ImportRequest) String() string { return proto.CompactTextString(m) } func (*ImportRequest) ProtoMessage() {} -func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{9} } - -func (m *ImportRequest) GetDB() string { - if m != nil && m.DB != nil { - return *m.DB - } - return "" -} - -func (m *ImportRequest) GetFrame() string { - if m != nil && m.Frame != nil { - return *m.Frame - } - return "" -} - -func (m *ImportRequest) GetSlice() uint64 { - if m != nil && m.Slice != nil { - return *m.Slice - } - return 0 -} - -func (m *ImportRequest) GetBitmapIDs() []uint64 { - if m != nil { - return m.BitmapIDs - } - return nil -} - -func (m *ImportRequest) GetProfileIDs() []uint64 { - if m != nil { - return m.ProfileIDs - } - return nil -} +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } type ImportResponse struct { - Err *string `protobuf:"bytes,1,opt,name=Err,json=err" json:"Err,omitempty"` - XXX_unrecognized []byte `json:"-"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` } func (m *ImportResponse) Reset() { *m = ImportResponse{} } func (m *ImportResponse) String() string { return proto.CompactTextString(m) } func (*ImportResponse) ProtoMessage() {} -func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{10} } - -func (m *ImportResponse) GetErr() string { - if m != nil && m.Err != nil { - return *m.Err - } - return "" -} +func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } type BlockDataRequest struct { - DB *string `protobuf:"bytes,1,req,name=DB,json=dB" json:"DB,omitempty"` - Frame *string `protobuf:"bytes,2,req,name=Frame,json=frame" json:"Frame,omitempty"` - Slice *uint64 `protobuf:"varint,3,req,name=Slice,json=slice" json:"Slice,omitempty"` - Block *uint64 `protobuf:"varint,4,req,name=Block,json=block" json:"Block,omitempty"` - XXX_unrecognized []byte `json:"-"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` } func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } func (m *BlockDataRequest) String() string { return proto.CompactTextString(m) } func (*BlockDataRequest) ProtoMessage() {} -func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } - -func (m *BlockDataRequest) GetDB() string { - if m != nil && m.DB != nil { - return *m.DB - } - return "" -} - -func (m *BlockDataRequest) GetFrame() string { - if m != nil && m.Frame != nil { - return *m.Frame - } - return "" -} - -func (m *BlockDataRequest) GetSlice() uint64 { - if m != nil && m.Slice != nil { - return *m.Slice - } - return 0 -} - -func (m *BlockDataRequest) GetBlock() uint64 { - if m != nil && m.Block != nil { - return *m.Block - } - return 0 -} +func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } type BlockDataResponse struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,2,rep,name=ProfileIDs,json=profileIDs" json:"ProfileIDs,omitempty"` - XXX_unrecognized []byte `json:"-"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` } func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } func (m *BlockDataResponse) String() string { return proto.CompactTextString(m) } func (*BlockDataResponse) ProtoMessage() {} -func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } - -func (m *BlockDataResponse) GetBitmapIDs() []uint64 { - if m != nil { - return m.BitmapIDs - } - return nil -} - -func (m *BlockDataResponse) GetProfileIDs() []uint64 { - if m != nil { - return m.ProfileIDs - } - return nil -} +func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } type Cache struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` - XXX_unrecognized []byte `json:"-"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` } func (m *Cache) Reset() { *m = Cache{} } func (m *Cache) String() string { return proto.CompactTextString(m) } func (*Cache) ProtoMessage() {} -func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } - -func (m *Cache) GetBitmapIDs() []uint64 { - if m != nil { - return m.BitmapIDs - } - return nil -} +func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{15} } type MaxSlicesResponse struct { - MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices,json=maxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"varint,2,opt,name=value"` - XXX_unrecognized []byte `json:"-"` + MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } func (m *MaxSlicesResponse) Reset() { *m = MaxSlicesResponse{} } func (m *MaxSlicesResponse) String() string { return proto.CompactTextString(m) } func (*MaxSlicesResponse) ProtoMessage() {} -func (*MaxSlicesResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } +func (*MaxSlicesResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{16} } func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 { if m != nil { @@ -506,6 +283,8 @@ func (m *MaxSlicesResponse) GetMaxSlices() map[string]uint64 { } func init() { + proto.RegisterType((*DB)(nil), "internal.DB") + proto.RegisterType((*Frame)(nil), "internal.Frame") proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*Bit)(nil), "internal.Bit") @@ -522,51 +301,3490 @@ func init() { proto.RegisterType((*Cache)(nil), "internal.Cache") proto.RegisterType((*MaxSlicesResponse)(nil), "internal.MaxSlicesResponse") } +func (m *DB) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} -func init() { proto.RegisterFile("internal/internal.proto", fileDescriptorInternal) } +func (m *DB) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } + return i, nil +} + +func (m *Frame) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Frame) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.TimeQuantum) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.TimeQuantum))) + i += copy(dAtA[i:], m.TimeQuantum) + } + return i, nil +} + +func (m *Bitmap) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Bitmap) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Bits) > 0 { + dAtA2 := make([]byte, len(m.Bits)*10) + var j1 int + for _, num := range m.Bits { + for num >= 1<<7 { + dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j1++ + } + dAtA2[j1] = uint8(num) + j1++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(j1)) + i += copy(dAtA[i:], dAtA2[:j1]) + } + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Pair) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Pair) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Key != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Key)) + } + if m.Count != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Count)) + } + return i, nil +} + +func (m *Bit) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Bit) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.BitmapID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.BitmapID)) + } + if m.ProfileID != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.ProfileID)) + } + return i, nil +} + +func (m *Profile) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Profile) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.ID != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.ID)) + } + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *Attr) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Attr) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Key) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Key))) + i += copy(dAtA[i:], m.Key) + } + if m.Type != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Type)) + } + if len(m.StringValue) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.StringValue))) + i += copy(dAtA[i:], m.StringValue) + } + if m.UintValue != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.UintValue)) + } + if m.BoolValue { + dAtA[i] = 0x28 + i++ + if m.BoolValue { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *AttrMap) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *AttrMap) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Attrs) > 0 { + for _, msg := range m.Attrs { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *QueryRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Query) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Query))) + i += copy(dAtA[i:], m.Query) + } + if len(m.Slices) > 0 { + dAtA4 := make([]byte, len(m.Slices)*10) + var j3 int + for _, num := range m.Slices { + for num >= 1<<7 { + dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j3++ + } + dAtA4[j3] = uint8(num) + j3++ + } + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(j3)) + i += copy(dAtA[i:], dAtA4[:j3]) + } + if m.Profiles { + dAtA[i] = 0x20 + i++ + if m.Profiles { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + if m.Timestamp != 0 { + dAtA[i] = 0x28 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Timestamp)) + } + if len(m.Quantum) > 0 { + dAtA[i] = 0x32 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Quantum))) + i += copy(dAtA[i:], m.Quantum) + } + if m.Remote { + dAtA[i] = 0x38 + i++ + if m.Remote { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *QueryResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Err) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Err))) + i += copy(dAtA[i:], m.Err) + } + if len(m.Results) > 0 { + for _, msg := range m.Results { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if len(m.Profiles) > 0 { + for _, msg := range m.Profiles { + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + return i, nil +} + +func (m *QueryResult) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.Bitmap != nil { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Bitmap.Size())) + n5, err := m.Bitmap.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n5 + } + if m.N != 0 { + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.N)) + } + if len(m.Pairs) > 0 { + for _, msg := range m.Pairs { + dAtA[i] = 0x1a + i++ + i = encodeVarintInternal(dAtA, i, uint64(msg.Size())) + n, err := msg.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n + } + } + if m.Changed { + dAtA[i] = 0x20 + i++ + if m.Changed { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } + return i, nil +} + +func (m *ImportRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Slice != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Slice)) + } + if len(m.BitmapIDs) > 0 { + dAtA7 := make([]byte, len(m.BitmapIDs)*10) + var j6 int + for _, num := range m.BitmapIDs { + for num >= 1<<7 { + dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j6++ + } + dAtA7[j6] = uint8(num) + j6++ + } + dAtA[i] = 0x22 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j6)) + i += copy(dAtA[i:], dAtA7[:j6]) + } + if len(m.ProfileIDs) > 0 { + dAtA9 := make([]byte, len(m.ProfileIDs)*10) + var j8 int + for _, num := range m.ProfileIDs { + for num >= 1<<7 { + dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j8++ + } + dAtA9[j8] = uint8(num) + j8++ + } + dAtA[i] = 0x2a + i++ + i = encodeVarintInternal(dAtA, i, uint64(j8)) + i += copy(dAtA[i:], dAtA9[:j8]) + } + return i, nil +} + +func (m *ImportResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ImportResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.Err) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Err))) + i += copy(dAtA[i:], m.Err) + } + return i, nil +} + +func (m *BlockDataRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockDataRequest) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.DB) > 0 { + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.DB))) + i += copy(dAtA[i:], m.DB) + } + if len(m.Frame) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(m.Frame))) + i += copy(dAtA[i:], m.Frame) + } + if m.Slice != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Slice)) + } + if m.Block != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Block)) + } + return i, nil +} + +func (m *BlockDataResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.BitmapIDs) > 0 { + dAtA11 := make([]byte, len(m.BitmapIDs)*10) + var j10 int + for _, num := range m.BitmapIDs { + for num >= 1<<7 { + dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j10++ + } + dAtA11[j10] = uint8(num) + j10++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(j10)) + i += copy(dAtA[i:], dAtA11[:j10]) + } + if len(m.ProfileIDs) > 0 { + dAtA13 := make([]byte, len(m.ProfileIDs)*10) + var j12 int + for _, num := range m.ProfileIDs { + for num >= 1<<7 { + dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j12++ + } + dAtA13[j12] = uint8(num) + j12++ + } + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j12)) + i += copy(dAtA[i:], dAtA13[:j12]) + } + return i, nil +} + +func (m *Cache) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Cache) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.BitmapIDs) > 0 { + dAtA15 := make([]byte, len(m.BitmapIDs)*10) + var j14 int + for _, num := range m.BitmapIDs { + for num >= 1<<7 { + dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j14++ + } + dAtA15[j14] = uint8(num) + j14++ + } + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(j14)) + i += copy(dAtA[i:], dAtA15[:j14]) + } + return i, nil +} + +func (m *MaxSlicesResponse) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *MaxSlicesResponse) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if len(m.MaxSlices) > 0 { + for k, _ := range m.MaxSlices { + dAtA[i] = 0xa + i++ + v := m.MaxSlices[k] + mapSize := 1 + len(k) + sovInternal(uint64(len(k))) + 1 + sovInternal(uint64(v)) + i = encodeVarintInternal(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintInternal(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + dAtA[i] = 0x10 + i++ + i = encodeVarintInternal(dAtA, i, uint64(v)) + } + } + return i, nil +} + +func encodeFixed64Internal(dAtA []byte, offset int, v uint64) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + dAtA[offset+4] = uint8(v >> 32) + dAtA[offset+5] = uint8(v >> 40) + dAtA[offset+6] = uint8(v >> 48) + dAtA[offset+7] = uint8(v >> 56) + return offset + 8 +} +func encodeFixed32Internal(dAtA []byte, offset int, v uint32) int { + dAtA[offset] = uint8(v) + dAtA[offset+1] = uint8(v >> 8) + dAtA[offset+2] = uint8(v >> 16) + dAtA[offset+3] = uint8(v >> 24) + return offset + 4 +} +func encodeVarintInternal(dAtA []byte, offset int, v uint64) int { + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return offset + 1 +} +func (m *DB) Size() (n int) { + var l int + _ = l + l = len(m.TimeQuantum) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + return n +} + +func (m *Frame) Size() (n int) { + var l int + _ = l + l = len(m.TimeQuantum) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + return n +} + +func (m *Bitmap) Size() (n int) { + var l int + _ = l + if len(m.Bits) > 0 { + l = 0 + for _, e := range m.Bits { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if len(m.Attrs) > 0 { + for _, e := range m.Attrs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *Pair) Size() (n int) { + var l int + _ = l + if m.Key != 0 { + n += 1 + sovInternal(uint64(m.Key)) + } + if m.Count != 0 { + n += 1 + sovInternal(uint64(m.Count)) + } + return n +} + +func (m *Bit) Size() (n int) { + var l int + _ = l + if m.BitmapID != 0 { + n += 1 + sovInternal(uint64(m.BitmapID)) + } + if m.ProfileID != 0 { + n += 1 + sovInternal(uint64(m.ProfileID)) + } + return n +} + +func (m *Profile) Size() (n int) { + var l int + _ = l + if m.ID != 0 { + n += 1 + sovInternal(uint64(m.ID)) + } + if len(m.Attrs) > 0 { + for _, e := range m.Attrs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *Attr) Size() (n int) { + var l int + _ = l + l = len(m.Key) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Type != 0 { + n += 1 + sovInternal(uint64(m.Type)) + } + l = len(m.StringValue) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.UintValue != 0 { + n += 1 + sovInternal(uint64(m.UintValue)) + } + if m.BoolValue { + n += 2 + } + return n +} + +func (m *AttrMap) Size() (n int) { + var l int + _ = l + if len(m.Attrs) > 0 { + for _, e := range m.Attrs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *QueryRequest) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + l = len(m.Query) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if len(m.Slices) > 0 { + l = 0 + for _, e := range m.Slices { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if m.Profiles { + n += 2 + } + if m.Timestamp != 0 { + n += 1 + sovInternal(uint64(m.Timestamp)) + } + l = len(m.Quantum) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Remote { + n += 2 + } + return n +} + +func (m *QueryResponse) Size() (n int) { + var l int + _ = l + l = len(m.Err) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if len(m.Results) > 0 { + for _, e := range m.Results { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + if len(m.Profiles) > 0 { + for _, e := range m.Profiles { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + return n +} + +func (m *QueryResult) Size() (n int) { + var l int + _ = l + if m.Bitmap != nil { + l = m.Bitmap.Size() + n += 1 + l + sovInternal(uint64(l)) + } + if m.N != 0 { + n += 1 + sovInternal(uint64(m.N)) + } + if len(m.Pairs) > 0 { + for _, e := range m.Pairs { + l = e.Size() + n += 1 + l + sovInternal(uint64(l)) + } + } + if m.Changed { + n += 2 + } + return n +} + +func (m *ImportRequest) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovInternal(uint64(m.Slice)) + } + if len(m.BitmapIDs) > 0 { + l = 0 + for _, e := range m.BitmapIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if len(m.ProfileIDs) > 0 { + l = 0 + for _, e := range m.ProfileIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + return n +} + +func (m *ImportResponse) Size() (n int) { + var l int + _ = l + l = len(m.Err) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + return n +} + +func (m *BlockDataRequest) Size() (n int) { + var l int + _ = l + l = len(m.DB) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + l = len(m.Frame) + if l > 0 { + n += 1 + l + sovInternal(uint64(l)) + } + if m.Slice != 0 { + n += 1 + sovInternal(uint64(m.Slice)) + } + if m.Block != 0 { + n += 1 + sovInternal(uint64(m.Block)) + } + return n +} + +func (m *BlockDataResponse) Size() (n int) { + var l int + _ = l + if len(m.BitmapIDs) > 0 { + l = 0 + for _, e := range m.BitmapIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + if len(m.ProfileIDs) > 0 { + l = 0 + for _, e := range m.ProfileIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + return n +} + +func (m *Cache) Size() (n int) { + var l int + _ = l + if len(m.BitmapIDs) > 0 { + l = 0 + for _, e := range m.BitmapIDs { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } + return n +} + +func (m *MaxSlicesResponse) Size() (n int) { + var l int + _ = l + if len(m.MaxSlices) > 0 { + for k, v := range m.MaxSlices { + _ = k + _ = v + mapEntrySize := 1 + len(k) + sovInternal(uint64(len(k))) + 1 + sovInternal(uint64(v)) + n += mapEntrySize + 1 + sovInternal(uint64(mapEntrySize)) + } + } + return n +} + +func sovInternal(x uint64) (n int) { + for { + n++ + x >>= 7 + if x == 0 { + break + } + } + return n +} +func sozInternal(x uint64) (n int) { + return sovInternal(uint64((x << 1) ^ uint64((int64(x) >> 63)))) +} +func (m *DB) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: DB: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: DB: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Frame) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Frame: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Frame: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeQuantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimeQuantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Bitmap) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Bitmap: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Bitmap: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Bits = append(m.Bits, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Bits = append(m.Bits, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType) + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attrs = append(m.Attrs, &Attr{}) + if err := m.Attrs[len(m.Attrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Pair) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Pair: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Pair: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + m.Key = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Key |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType) + } + m.Count = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Count |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Bit) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Bit: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Bit: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapID", wireType) + } + m.BitmapID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.BitmapID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ProfileID", wireType) + } + m.ProfileID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ProfileID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Profile) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Profile: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Profile: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ID", wireType) + } + m.ID = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ID |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attrs = append(m.Attrs, &Attr{}) + if err := m.Attrs[len(m.Attrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Attr) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Attr: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Attr: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Key", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Key = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Type", wireType) + } + m.Type = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Type |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StringValue", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.StringValue = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field UintValue", wireType) + } + m.UintValue = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.UintValue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field BoolValue", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BoolValue = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *AttrMap) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: AttrMap: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: AttrMap: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attrs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attrs = append(m.Attrs, &Attr{}) + if err := m.Attrs[len(m.Attrs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Query", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Query = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Slices = append(m.Slices, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Slices", wireType) + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Profiles = bool(v != 0) + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) + } + m.Timestamp = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Timestamp |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Quantum", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Quantum = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Remote", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Remote = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Err", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Err = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Results", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Results = append(m.Results, &QueryResult{}) + if err := m.Results[len(m.Results)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Profiles = append(m.Profiles, &Profile{}) + if err := m.Profiles[len(m.Profiles)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *QueryResult) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: QueryResult: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: QueryResult: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Bitmap", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Bitmap == nil { + m.Bitmap = &Bitmap{} + } + if err := m.Bitmap.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field N", wireType) + } + m.N = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.N |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Pairs", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Pairs = append(m.Pairs, &Pair{}) + if err := m.Pairs[len(m.Pairs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Changed", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Changed = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ImportRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + } + case 5: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ImportResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ImportResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ImportResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Err", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Err = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockDataRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockDataRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DB", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DB = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Frame", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Frame = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Slice", wireType) + } + m.Slice = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Slice |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Block", wireType) + } + m.Block = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Block |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: BlockDataResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: BlockDataResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + } + case 2: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.ProfileIDs = append(m.ProfileIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *Cache) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Cache: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Cache: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } + } else if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.BitmapIDs = append(m.BitmapIDs, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *MaxSlicesResponse) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: MaxSlicesResponse: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: MaxSlicesResponse: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field MaxSlices", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + msglen + if postIndex > l { + return io.ErrUnexpectedEOF + } + var keykey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + keykey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthInternal + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey := string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + if m.MaxSlices == nil { + m.MaxSlices = make(map[string]uint64) + } + if iNdEx < postIndex { + var valuekey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + valuekey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + var mapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapvalue |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.MaxSlices[mapkey] = mapvalue + } else { + var mapvalue uint64 + m.MaxSlices[mapkey] = mapvalue + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipInternal(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthInternal + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func skipInternal(dAtA []byte) (n int, err error) { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + wireType := int(wire & 0x7) + switch wireType { + case 0: + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + iNdEx++ + if dAtA[iNdEx-1] < 0x80 { + break + } + } + return iNdEx, nil + case 1: + iNdEx += 8 + return iNdEx, nil + case 2: + var length int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + length |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + iNdEx += length + if length < 0 { + return 0, ErrInvalidLengthInternal + } + return iNdEx, nil + case 3: + for { + var innerWire uint64 + var start int = iNdEx + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return 0, ErrIntOverflowInternal + } + if iNdEx >= l { + return 0, io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + innerWire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + innerWireType := int(innerWire & 0x7) + if innerWireType == 4 { + break + } + next, err := skipInternal(dAtA[start:]) + if err != nil { + return 0, err + } + iNdEx = start + next + } + return iNdEx, nil + case 4: + return iNdEx, nil + case 5: + iNdEx += 4 + return iNdEx, nil + default: + return 0, fmt.Errorf("proto: illegal wireType %d", wireType) + } + } + panic("unreachable") +} + +var ( + ErrInvalidLengthInternal = fmt.Errorf("proto: negative length found during unmarshaling") + ErrIntOverflowInternal = fmt.Errorf("proto: integer overflow") +) + +func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 667 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x54, 0x5f, 0x6b, 0xd4, 0x4e, - 0x14, 0x65, 0xf2, 0x67, 0xb3, 0xb9, 0xdb, 0x2e, 0xed, 0xd0, 0xdf, 0xcf, 0x50, 0x44, 0x42, 0x50, - 0x08, 0x82, 0x2d, 0xf4, 0x49, 0x44, 0x28, 0xa6, 0x5b, 0xb1, 0x48, 0xa5, 0x9d, 0xaa, 0xef, 0xb3, - 0xdb, 0x69, 0x1b, 0x9a, 0x7f, 0x9d, 0x99, 0x14, 0xf7, 0xd1, 0x07, 0x1f, 0xfc, 0x0c, 0x7e, 0x11, - 0x3f, 0x9e, 0xcc, 0x64, 0x26, 0xd9, 0x52, 0xac, 0x3e, 0xf8, 0x96, 0x73, 0x6e, 0xee, 0xdc, 0x73, - 0xef, 0x99, 0x3b, 0xf0, 0x28, 0xaf, 0x24, 0xe3, 0x15, 0x2d, 0x76, 0xed, 0xc7, 0x4e, 0xc3, 0x6b, - 0x59, 0xe3, 0xb1, 0xc5, 0x49, 0x06, 0xa3, 0x2c, 0x97, 0x25, 0x6d, 0x30, 0x06, 0x2f, 0xcb, 0xa5, - 0x88, 0x50, 0xec, 0xa6, 0x1e, 0xf1, 0xe6, 0xb9, 0x14, 0xf8, 0x29, 0xf8, 0x6f, 0xa4, 0xe4, 0x22, - 0x72, 0x62, 0x37, 0x9d, 0xec, 0x4d, 0x77, 0xfa, 0x73, 0x14, 0x4d, 0x7c, 0xaa, 0x82, 0xc9, 0x0e, - 0x78, 0x27, 0x34, 0xe7, 0x78, 0x03, 0xdc, 0xf7, 0x6c, 0x19, 0xa1, 0xd8, 0x49, 0x3d, 0xe2, 0x5e, - 0xb3, 0x25, 0xde, 0x02, 0xff, 0xa0, 0x6e, 0x2b, 0x19, 0x39, 0x9a, 0xf3, 0x17, 0x0a, 0x24, 0xfb, - 0xe0, 0x66, 0xb9, 0xc4, 0xdb, 0x30, 0xee, 0x4a, 0x1f, 0xcd, 0x4c, 0xce, 0x78, 0x6e, 0x30, 0x7e, - 0x0c, 0xe1, 0x09, 0xaf, 0x2f, 0xf2, 0x82, 0x1d, 0xcd, 0x4c, 0x72, 0xd8, 0x58, 0x22, 0xd9, 0x87, - 0xc0, 0x44, 0xf1, 0x14, 0x9c, 0x3e, 0xdd, 0xc9, 0x67, 0x7f, 0xa9, 0xf8, 0x16, 0x3c, 0x05, 0x57, - 0x15, 0x87, 0x9d, 0xe2, 0x18, 0x26, 0x67, 0x92, 0xe7, 0xd5, 0xe5, 0x67, 0x5a, 0xb4, 0x2c, 0x72, - 0x62, 0x94, 0x86, 0x64, 0x22, 0x06, 0x4a, 0x49, 0xfb, 0x94, 0x57, 0xb2, 0x8b, 0xbb, 0x31, 0x52, - 0xd2, 0x5a, 0x4b, 0xa8, 0x68, 0x56, 0xd7, 0x45, 0x17, 0xf5, 0x62, 0x94, 0x8e, 0x49, 0x38, 0xb7, - 0x44, 0xb2, 0x0b, 0x81, 0xaa, 0x7b, 0x4c, 0x9b, 0x41, 0x28, 0x7a, 0x48, 0xe8, 0x4f, 0x04, 0x6b, - 0xa7, 0x2d, 0xe3, 0x4b, 0xc2, 0x6e, 0x5a, 0x26, 0xa4, 0xea, 0x77, 0x96, 0x19, 0xc1, 0xce, 0x79, - 0xa6, 0x26, 0xac, 0xe3, 0x7a, 0x48, 0x21, 0xf1, 0x6f, 0x14, 0xc0, 0xff, 0xc3, 0xe8, 0xac, 0xc8, - 0x17, 0x4c, 0x44, 0xae, 0x76, 0x73, 0x24, 0x34, 0x52, 0x23, 0x37, 0x83, 0x13, 0x46, 0xdc, 0xd8, - 0x4c, 0x55, 0x28, 0xe5, 0x1f, 0xf3, 0x92, 0x09, 0x49, 0xcb, 0x26, 0xf2, 0x63, 0x94, 0xba, 0x24, - 0x94, 0x96, 0xc0, 0x11, 0x04, 0xa7, 0x2d, 0xad, 0x64, 0x5b, 0x46, 0xa3, 0x18, 0xa5, 0xeb, 0x24, - 0xb8, 0xe9, 0xa0, 0xaa, 0x45, 0x58, 0x59, 0x4b, 0x16, 0x05, 0xfa, 0xc4, 0x11, 0xd7, 0x28, 0xf9, - 0x8a, 0x60, 0xdd, 0x48, 0x17, 0x4d, 0x5d, 0x09, 0xa6, 0xa6, 0x7d, 0xc8, 0x79, 0x84, 0xf4, 0x4c, - 0x5d, 0xc6, 0x39, 0xde, 0x85, 0x80, 0x30, 0xd1, 0x16, 0xd2, 0xfa, 0xf5, 0xdf, 0x30, 0x06, 0x9b, - 0xdb, 0x16, 0x92, 0x04, 0xbc, 0xfb, 0x0b, 0xbf, 0x58, 0x69, 0xc0, 0xd5, 0x19, 0x9b, 0x43, 0x86, - 0x89, 0x0c, 0x3d, 0x25, 0xdf, 0x10, 0x4c, 0x56, 0xce, 0xc1, 0xa9, 0xbd, 0xed, 0x5a, 0xc4, 0x64, - 0x6f, 0x63, 0x48, 0xee, 0x78, 0x32, 0xea, 0xae, 0x20, 0x5e, 0x03, 0xf4, 0x41, 0xbb, 0xef, 0x11, - 0x54, 0x29, 0xb3, 0xd4, 0x0d, 0xb7, 0x35, 0x57, 0xcc, 0x52, 0x34, 0xf1, 0x1b, 0x15, 0x54, 0x33, - 0x3a, 0xb8, 0xa2, 0xd5, 0x25, 0x3b, 0x37, 0xc3, 0x0d, 0x16, 0x1d, 0x4c, 0xbe, 0x23, 0x58, 0x3f, - 0x2a, 0x9b, 0x9a, 0xcb, 0x07, 0x7c, 0x7c, 0xcb, 0x69, 0xc9, 0xac, 0x8f, 0x17, 0x0a, 0x28, 0x56, - 0xfb, 0x18, 0xb9, 0xdd, 0xfe, 0x68, 0x1b, 0xf5, 0x1d, 0x33, 0x8b, 0xa2, 0x6c, 0x54, 0x06, 0x87, - 0x76, 0x73, 0x04, 0x7e, 0x02, 0xd0, 0xaf, 0x8e, 0x88, 0x7c, 0x1d, 0x86, 0x7e, 0x77, 0x44, 0x92, - 0xc0, 0xd4, 0x4a, 0xf9, 0x9d, 0x2f, 0xc9, 0x39, 0x6c, 0x64, 0x45, 0xbd, 0xb8, 0x9e, 0x51, 0x49, - 0xff, 0x85, 0xe2, 0x2d, 0xf0, 0xf5, 0x79, 0x91, 0xd7, 0xb1, 0x73, 0x05, 0x92, 0x53, 0xd8, 0x5c, - 0xa9, 0x62, 0xc4, 0xdc, 0x69, 0x0e, 0x3d, 0xdc, 0x9c, 0x73, 0xaf, 0xb9, 0x67, 0xe0, 0x1f, 0xd0, - 0xc5, 0xd5, 0x1f, 0x8e, 0x49, 0x7e, 0x20, 0xd8, 0x3c, 0xa6, 0x5f, 0xba, 0x1d, 0xe9, 0x4b, 0xbf, - 0x83, 0xb0, 0x27, 0xcd, 0x5a, 0x3e, 0x1f, 0x9c, 0xbe, 0xf7, 0xff, 0xc0, 0x1c, 0x56, 0x92, 0x2f, - 0x49, 0x58, 0x5a, 0xbc, 0xfd, 0x1a, 0xa6, 0x77, 0x83, 0x6a, 0xc6, 0xd7, 0xfa, 0xa5, 0x41, 0xf6, - 0xa5, 0xd9, 0x02, 0xff, 0xb6, 0x7f, 0x63, 0x3c, 0xd2, 0x81, 0x57, 0xce, 0x4b, 0xf4, 0x2b, 0x00, - 0x00, 0xff, 0xff, 0x1a, 0xd6, 0x56, 0x91, 0xb7, 0x05, 0x00, 0x00, + // 682 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6a, 0x14, 0x41, + 0x10, 0xb6, 0x77, 0x66, 0xf6, 0xa7, 0x36, 0x59, 0x36, 0xcd, 0x2a, 0x43, 0x90, 0x65, 0x69, 0x54, + 0x56, 0xc1, 0x04, 0xe2, 0x45, 0x44, 0x08, 0x4e, 0x36, 0xe2, 0x22, 0x09, 0x49, 0x27, 0x7a, 0x1f, + 0x93, 0x36, 0x19, 0x32, 0x7f, 0xf6, 0xf4, 0x88, 0x7b, 0xf4, 0xe0, 0x41, 0x9f, 0x40, 0xf0, 0x45, + 0x7c, 0x04, 0x8f, 0x3e, 0x82, 0xc4, 0x17, 0x91, 0xfe, 0x99, 0xe9, 0x09, 0xc1, 0x98, 0x83, 0xb7, + 0xae, 0xaf, 0xa6, 0xba, 0xbe, 0xaa, 0xaf, 0xaa, 0x07, 0x06, 0x51, 0x2a, 0x18, 0x4f, 0xc3, 0x78, + 0x2d, 0xe7, 0x99, 0xc8, 0x70, 0xb7, 0xb2, 0xc9, 0x3d, 0x68, 0xcd, 0x02, 0x3c, 0x81, 0xfe, 0x61, + 0x94, 0xb0, 0xfd, 0x32, 0x4c, 0x45, 0x99, 0xf8, 0x68, 0x82, 0xa6, 0x3d, 0xda, 0x84, 0xc8, 0x7d, + 0xf0, 0x9e, 0xf3, 0x30, 0x61, 0xd7, 0xf8, 0x34, 0x80, 0x76, 0x10, 0x89, 0x24, 0xcc, 0x31, 0x06, + 0x37, 0x88, 0x44, 0xe1, 0xa3, 0x89, 0x33, 0x75, 0xa9, 0x3a, 0xe3, 0x3b, 0xe0, 0x3d, 0x13, 0x82, + 0x17, 0x7e, 0x6b, 0xe2, 0x4c, 0xfb, 0x1b, 0x83, 0xb5, 0x9a, 0x9a, 0x84, 0xa9, 0x76, 0x92, 0x35, + 0x70, 0xf7, 0xc2, 0x88, 0xe3, 0x21, 0x38, 0x2f, 0xd9, 0x42, 0x65, 0x71, 0xa9, 0x3c, 0xe2, 0x11, + 0x78, 0x5b, 0x59, 0x99, 0x0a, 0xbf, 0xa5, 0x30, 0x6d, 0x90, 0x4d, 0x70, 0x82, 0x48, 0xe0, 0x55, + 0xe8, 0xea, 0xd4, 0xf3, 0x99, 0x89, 0xa9, 0x6d, 0x7c, 0x1b, 0x7a, 0x7b, 0x3c, 0x7b, 0x1b, 0xc5, + 0x6c, 0x3e, 0x33, 0xc1, 0x16, 0x20, 0x9b, 0xd0, 0x31, 0x06, 0x1e, 0x40, 0xab, 0x0e, 0x6f, 0xcd, + 0x67, 0xd7, 0x64, 0xfc, 0x05, 0x81, 0x2b, 0x4f, 0x4d, 0xca, 0x3d, 0x4d, 0x19, 0x83, 0x7b, 0xb8, + 0xc8, 0x99, 0x49, 0xaa, 0xce, 0xb2, 0x8d, 0x07, 0x82, 0x47, 0xe9, 0xc9, 0xeb, 0x30, 0x2e, 0x99, + 0xef, 0xe8, 0x36, 0x36, 0x20, 0xc9, 0xf7, 0x55, 0x94, 0x0a, 0xed, 0x77, 0x35, 0xdf, 0x1a, 0x90, + 0xde, 0x20, 0xcb, 0x62, 0xed, 0xf5, 0x26, 0x68, 0xda, 0xa5, 0x16, 0x20, 0xeb, 0xd0, 0x91, 0x5c, + 0x76, 0xc2, 0xdc, 0xb2, 0x47, 0x57, 0xb1, 0xff, 0x8e, 0x60, 0x69, 0xbf, 0x64, 0x7c, 0x41, 0xd9, + 0xbb, 0x92, 0x15, 0x42, 0x36, 0x61, 0x16, 0x98, 0x22, 0xe4, 0x84, 0x8c, 0xc0, 0x53, 0x7e, 0x55, + 0x44, 0x8f, 0x6a, 0x03, 0xdf, 0x82, 0xf6, 0x41, 0x1c, 0x1d, 0xb1, 0xc2, 0x77, 0x94, 0xc4, 0xc6, + 0x92, 0x3a, 0x98, 0x6e, 0x16, 0x8a, 0x7a, 0x97, 0xd6, 0xb6, 0x64, 0x2e, 0xa7, 0xa5, 0x10, 0x61, + 0x92, 0x2b, 0xe6, 0x0e, 0xb5, 0x00, 0xf6, 0xa1, 0x53, 0x8d, 0x56, 0x5b, 0x65, 0xaa, 0x4c, 0x99, + 0x8b, 0xb2, 0x24, 0x13, 0xcc, 0xef, 0xa8, 0x1b, 0x8d, 0x45, 0x3e, 0x22, 0x58, 0x36, 0xd4, 0x8b, + 0x3c, 0x4b, 0x0b, 0x26, 0x15, 0xd8, 0xe6, 0xbc, 0x52, 0x60, 0x9b, 0x73, 0xbc, 0x0e, 0x1d, 0xca, + 0x8a, 0x32, 0x16, 0x95, 0x88, 0x37, 0x6d, 0x1b, 0xaa, 0xd8, 0x32, 0x16, 0xb4, 0xfa, 0x0a, 0x3f, + 0x6c, 0x14, 0xe0, 0xa8, 0x88, 0x15, 0x1b, 0x61, 0x3c, 0xb6, 0x26, 0xf2, 0x09, 0x41, 0xbf, 0x71, + 0x0f, 0x9e, 0x56, 0x2b, 0xa0, 0x48, 0xf4, 0x37, 0x86, 0x36, 0x58, 0xe3, 0xb4, 0x5a, 0x91, 0x25, + 0x40, 0xbb, 0x66, 0x30, 0xd0, 0xae, 0x14, 0x4b, 0x8e, 0x7d, 0x95, 0xb3, 0x21, 0x96, 0x84, 0xa9, + 0x76, 0xca, 0x1e, 0x6d, 0x9d, 0x86, 0xe9, 0x09, 0x3b, 0x36, 0xcd, 0xad, 0x4c, 0xf2, 0x19, 0xc1, + 0xf2, 0x3c, 0xc9, 0x33, 0x2e, 0xae, 0xd0, 0x51, 0xed, 0x71, 0xa5, 0xa3, 0x5e, 0xea, 0x11, 0x78, + 0x4a, 0x39, 0x35, 0x87, 0x2e, 0xd5, 0x86, 0x9a, 0x31, 0xb3, 0x3d, 0x52, 0x46, 0x29, 0xb0, 0x05, + 0xf0, 0x18, 0xa0, 0x5e, 0x9f, 0xc2, 0xf7, 0x94, 0xbb, 0x81, 0x10, 0x02, 0x83, 0x8a, 0xca, 0xdf, + 0x74, 0x21, 0xc7, 0x30, 0x0c, 0xe2, 0xec, 0xe8, 0x6c, 0x16, 0x8a, 0xf0, 0x7f, 0x30, 0x1e, 0x81, + 0xa7, 0xee, 0x33, 0xfb, 0xa2, 0x0d, 0xb2, 0x0f, 0x2b, 0x8d, 0x2c, 0x86, 0xcc, 0x85, 0xe2, 0xd0, + 0xd5, 0xc5, 0xb5, 0x2e, 0x15, 0x77, 0x17, 0xbc, 0xad, 0xf0, 0xe8, 0xf4, 0x1f, 0xd7, 0x90, 0x6f, + 0x08, 0x56, 0x76, 0xc2, 0x0f, 0x7a, 0x2b, 0xea, 0xd4, 0x2f, 0xa0, 0x57, 0x83, 0x66, 0x2d, 0x1f, + 0x58, 0xa5, 0x2f, 0x7d, 0x6f, 0x91, 0xed, 0x54, 0xf0, 0x05, 0xb5, 0xc1, 0xab, 0x4f, 0x61, 0x70, + 0xd1, 0x29, 0x7b, 0x7c, 0x66, 0x5f, 0x9f, 0x33, 0xfd, 0x60, 0xbe, 0x57, 0xaf, 0x84, 0x79, 0x30, + 0x95, 0xf1, 0xa4, 0xf5, 0x18, 0x05, 0xc3, 0x1f, 0xe7, 0x63, 0xf4, 0xf3, 0x7c, 0x8c, 0x7e, 0x9d, + 0x8f, 0xd1, 0xd7, 0xdf, 0xe3, 0x1b, 0x6f, 0xda, 0xea, 0xf7, 0xf0, 0xe8, 0x4f, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x49, 0xf3, 0xc1, 0xa7, 0x30, 0x06, 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index 6c2990360..a26cd1f02 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -1,31 +1,41 @@ -syntax = "proto2"; +syntax = "proto3"; + package internal; +message DB { + string TimeQuantum = 1; +} + +message Frame { + string TimeQuantum = 1; +} + message Bitmap { - repeated uint64 Bits = 1; - repeated Attr Attrs = 2; + repeated uint64 Bits = 1; + repeated Attr Attrs = 2; } message Pair { - required uint64 Key = 1; - required uint64 Count = 2; + uint64 Key = 1; + uint64 Count = 2; } message Bit { - required uint64 BitmapID = 1; - required uint64 ProfileID = 2; + uint64 BitmapID = 1; + uint64 ProfileID = 2; } message Profile { - required uint64 ID = 1; + uint64 ID = 1; repeated Attr Attrs = 2; } message Attr { - required string Key = 1; - optional string StringValue = 2; - optional uint64 UintValue = 3; - optional bool BoolValue = 4; + string Key = 1; + uint64 Type = 2; + string StringValue = 3; + uint64 UintValue = 4; + bool BoolValue = 5; } message AttrMap { @@ -33,49 +43,49 @@ message AttrMap { } message QueryRequest { - required string DB = 1; - required string Query = 2; - repeated uint64 Slices = 3; - optional bool Profiles = 4; - optional int64 Timestamp = 5; - optional uint32 Quantum = 6; - optional bool Remote = 7; + string DB = 1; + string Query = 2; + repeated uint64 Slices = 3; + bool Profiles = 4; + int64 Timestamp = 5; + string Quantum = 6; + bool Remote = 7; } message QueryResponse { - optional string Err = 1; - repeated QueryResult Results = 2; - repeated Profile Profiles = 3; + string Err = 1; + repeated QueryResult Results = 2; + repeated Profile Profiles = 3; } message QueryResult { - optional Bitmap Bitmap = 1; - optional uint64 N = 2; - repeated Pair Pairs = 3; - optional bool Changed = 4; + Bitmap Bitmap = 1; + uint64 N = 2; + repeated Pair Pairs = 3; + bool Changed = 4; } message ImportRequest { - required string DB = 1; - required string Frame = 2; - required uint64 Slice = 3; - repeated uint64 BitmapIDs = 4; + string DB = 1; + string Frame = 2; + uint64 Slice = 3; + repeated uint64 BitmapIDs = 4; repeated uint64 ProfileIDs = 5; } message ImportResponse { - optional string Err = 1; + string Err = 1; } message BlockDataRequest { - required string DB = 1; - required string Frame = 2; - required uint64 Slice = 3; - required uint64 Block = 4; + string DB = 1; + string Frame = 2; + uint64 Slice = 3; + uint64 Block = 4; } message BlockDataResponse { - repeated uint64 BitmapIDs = 1; + repeated uint64 BitmapIDs = 1; repeated uint64 ProfileIDs = 2; } @@ -86,3 +96,4 @@ message Cache { message MaxSlicesResponse { map MaxSlices = 1; } + diff --git a/pilosa.go b/pilosa.go index 4d1fbba41..d6c07bb0c 100644 --- a/pilosa.go +++ b/pilosa.go @@ -3,12 +3,9 @@ package pilosa import ( "errors" - "github.com/gogo/protobuf/proto" "github.com/pilosa/pilosa/internal" ) -//go:generate protoc --gogo_out=. internal/internal.proto - var ( // ErrHostRequired is returned when excuting a remote operation without a host. ErrHostRequired = errors.New("host required") @@ -57,7 +54,7 @@ func decodeProfiles(a []*internal.Profile) []*Profile { // encodeProfile converts p into its internal representation. func encodeProfile(p *Profile) *internal.Profile { return &internal.Profile{ - ID: proto.Uint64(p.ID), + ID: p.ID, Attrs: encodeAttrs(p.Attrs), } } @@ -65,12 +62,12 @@ func encodeProfile(p *Profile) *internal.Profile { // decodeProfile converts b from its internal representation. func decodeProfile(pb *internal.Profile) *Profile { p := &Profile{ - ID: pb.GetID(), + ID: pb.ID, } - if len(pb.GetAttrs()) > 0 { - p.Attrs = make(map[string]interface{}, len(pb.GetAttrs())) - for _, attr := range pb.GetAttrs() { + if len(pb.Attrs) > 0 { + p.Attrs = make(map[string]interface{}, len(pb.Attrs)) + for _, attr := range pb.Attrs { k, v := decodeAttr(attr) p.Attrs[k] = v } diff --git a/server.go b/server.go index 7323935fe..3dd853f01 100644 --- a/server.go +++ b/server.go @@ -266,5 +266,4 @@ func checkMaxSlices(hostport string) (map[string]uint64, error) { } return pb.MaxSlices, nil - } diff --git a/time.go b/time.go index ce11ea1d9..b117023ad 100644 --- a/time.go +++ b/time.go @@ -2,122 +2,166 @@ package pilosa import ( "errors" + "fmt" "strings" "time" ) -// TimeQuantum represents a time granularity for time-based bitmap ids. -type TimeQuantum uint +// ErrInvalidTimeQuantum is returned when parsing a time quantum. +var ErrInvalidTimeQuantum = errors.New("invalid time quantum") -// ParseTimeQuantum parses s into a quantum. -func ParseTimeQuantum(s string) (TimeQuantum, error) { - switch strings.ToUpper(s) { - case "Y": - return Y, nil - case "M": - return YM, nil - case "D": - return YMD, nil - case "H": - return YMDH, nil - default: - return 0, errors.New("invalid quantum") - } -} +// TimeQuantum represents a time granularity for time-based bitmaps. +type TimeQuantum string -// String returns the string representation of the quantum. -func (q TimeQuantum) String() string { +// HasYear returns true if the quantum contains a 'Y' unit. +func (q TimeQuantum) HasYear() bool { return strings.ContainsRune(string(q), 'Y') } + +// HasMonth returns true if the quantum contains a 'M' unit. +func (q TimeQuantum) HasMonth() bool { return strings.ContainsRune(string(q), 'M') } + +// HasDay returns true if the quantum contains a 'D' unit. +func (q TimeQuantum) HasDay() bool { return strings.ContainsRune(string(q), 'D') } + +// HasHour returns true if the quantum contains a 'H' unit. +func (q TimeQuantum) HasHour() bool { return strings.ContainsRune(string(q), 'H') } + +// Valid returns true if q is a valid time quantum value. +func (q TimeQuantum) Valid() bool { switch q { - case Y: - return "Y" - case YM: - return "M" - case YMD: - return "D" - case YMDH: - return "H" + case "Y", "YM", "YMD", "YMDH", + "M", "MD", "MDH", + "D", "DH", + "H", + "": + return true default: - return "Y" + return false } } -const ( - Y TimeQuantum = 3 - YM TimeQuantum = 2 - YMD TimeQuantum = 1 - YMDH TimeQuantum = 0 -) - -// TimeID returns a packed time identifier from a year/month/day/hour & tile. -func TimeID(q TimeQuantum, year uint, month uint, day uint, hour uint, tileID uint64) uint64 { - v := uint64((uint(q) << 30) | ((year - 1970) << 23) | (month << 19) | (day << 14) | (hour << 9)) - return (v << 32) | tileID +// ParseTimeQuantum parses v into a time quantum. +func ParseTimeQuantum(v string) (TimeQuantum, error) { + q := TimeQuantum(strings.ToUpper(v)) + if !q.Valid() { + return "", ErrInvalidTimeQuantum + } + return q, nil } -// TimeIDsFromRange returns timestamp bitmap ids within a time range. -func TimeIDsFromRange(start, end time.Time, tileID uint64) []uint64 { +// FrameByTimeUnit returns the frame name for time with a given quantum unit. +func FrameByTimeUnit(name string, t time.Time, unit rune) string { + switch unit { + case 'Y': + return fmt.Sprintf("%s_%s", name, t.Format("2006")) + case 'M': + return fmt.Sprintf("%s_%s", name, t.Format("200601")) + case 'D': + return fmt.Sprintf("%s_%s", name, t.Format("20060102")) + case 'H': + return fmt.Sprintf("%s_%s", name, t.Format("2006010215")) + default: + return "" + } +} + +// FramesByTime returns a list of frames for a given timestamp. +func FramesByTime(name string, t time.Time, q TimeQuantum) []string { + a := make([]string, 0, len(q)) + for _, unit := range q { + frame := FrameByTimeUnit(name, t, unit) + if frame == "" { + continue + } + a = append(a, frame) + } + return a +} + +// FramesByTimeRange returns a list of frames to traverse to query a time range. +func FramesByTimeRange(name string, start, end time.Time, q TimeQuantum) []string { t := start - var results []uint64 - for t.Before(end) { - if !nextDay(t, end) { + // Save flags for performance. + hasYear := q.HasYear() + hasMonth := q.HasMonth() + hasDay := q.HasDay() + hasHour := q.HasHour() + + var results []string + + // Walk up from smallest units to largest units. + if hasHour || hasDay || hasMonth { + for t.Before(end) { + if hasHour { + if !nextDayGTE(t, end) { + break + } else if t.Hour() != 0 { + results = append(results, FrameByTimeUnit(name, t, 'H')) + t = t.Add(time.Hour) + continue + } + + } + + if hasDay { + if !nextMonthGTE(t, end) { + break + } else if t.Day() != 1 { + results = append(results, FrameByTimeUnit(name, t, 'D')) + t = t.AddDate(0, 0, 1) + continue + } + } + + if hasMonth { + if !nextYearGTE(t, end) { + break + } else if t.Month() != 1 { + results = append(results, FrameByTimeUnit(name, t, 'M')) + t = t.AddDate(0, 1, 0) + continue + } + } + + // If a unit exists but isn't set and there are no larger units + // available then we need to exit the loop because we are no longer + // making progress. break } - if t.Hour() == 0 { - if !nextMonth(t, end) { - break - } - - if t.Day() == 1 { - if !nextYear(t, end) { - break - } - - if t.Month() == 1 { - break - } - - results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) - t = t.AddDate(0, 1, 0) - } else { - results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) - t = t.AddDate(0, 0, 1) - } - } else { - results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) - t = t.Add(time.Hour) - } } + // Walk back down from largest units to smallest units. for t.Before(end) { - if nextYear(t, end) { - results = append(results, TimeID(Y, uint(t.Year()), 0, 0, 0, tileID)) + if hasYear && nextYearGTE(t, end) { + results = append(results, FrameByTimeUnit(name, t, 'Y')) t = t.AddDate(1, 0, 0) - } else if nextMonth(t, end) { - results = append(results, TimeID(YM, uint(t.Year()), uint(t.Month()), 0, 0, tileID)) + } else if hasMonth && nextMonthGTE(t, end) { + results = append(results, FrameByTimeUnit(name, t, 'M')) t = t.AddDate(0, 1, 0) - } else if nextDay(t, end) { - results = append(results, TimeID(YMD, uint(t.Year()), uint(t.Month()), uint(t.Day()), 0, tileID)) + } else if hasDay && nextDayGTE(t, end) { + results = append(results, FrameByTimeUnit(name, t, 'D')) t = t.AddDate(0, 0, 1) - } else { - results = append(results, TimeID(YMDH, uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()), tileID)) + } else if hasHour { + results = append(results, FrameByTimeUnit(name, t, 'H')) t = t.Add(time.Hour) + } else { + break } } return results } -func nextYear(start time.Time, end time.Time) bool { - next := start.AddDate(1, 0, 0) +func nextYearGTE(t time.Time, end time.Time) bool { + next := t.AddDate(1, 0, 0) if next.Year() == end.Year() { return true } return end.After(next) } -func nextMonth(start time.Time, end time.Time) bool { - next := start.AddDate(0, 1, 0) +func nextMonthGTE(t time.Time, end time.Time) bool { + next := t.AddDate(0, 1, 0) y1, m1, _ := next.Date() y2, m2, _ := end.Date() if (y1 == y2) && (m1 == m2) { @@ -126,8 +170,8 @@ func nextMonth(start time.Time, end time.Time) bool { return end.After(next) } -func nextDay(start time.Time, end time.Time) bool { - next := start.AddDate(0, 0, 1) +func nextDayGTE(t time.Time, end time.Time) bool { + next := t.AddDate(0, 0, 1) y1, m1, d1 := next.Date() y2, m2, d2 := end.Date() if (y1 == y2) && (m1 == m2) && (d1 == d2) { @@ -135,23 +179,3 @@ func nextDay(start time.Time, end time.Time) bool { } return end.After(next) } - -// TimeIDsFromQuantum returns a list of time identifiers for a single quantum. -func TimeIDsFromQuantum(q TimeQuantum, t time.Time, tileID uint64) []uint64 { - y, m, d, h := uint(t.Year()), uint(t.Month()), uint(t.Day()), uint(t.Hour()) - - v := make([]uint64, 0, 4) - if q <= Y { - v = append(v, TimeID(Y, y, 0, 0, 0, tileID)) - } - if q <= YM { - v = append(v, TimeID(YM, y, m, 0, 0, tileID)) - } - if q <= YMD { - v = append(v, TimeID(YMD, y, m, d, 0, tileID)) - } - if q <= YMDH { - v = append(v, TimeID(YMDH, y, m, d, h, tileID)) - } - return v -} diff --git a/time_test.go b/time_test.go index aed62849c..28d86555d 100644 --- a/time_test.go +++ b/time_test.go @@ -1,134 +1,162 @@ package pilosa_test import ( + "reflect" "testing" "time" "github.com/pilosa/pilosa" ) -func TestTimeIDsFromRange_1h_0(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-08-11 14:00"), - *MustParseTime("2014-08-11 16:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } +// Ensure string can be parsed into time quantum. +func TestParseTimeQuantum(t *testing.T) { + t.Run("OK", func(t *testing.T) { + if q, err := pilosa.ParseTimeQuantum("YMDH"); err != nil { + t.Fatalf("unexpected error: %s", err) + } else if q != pilosa.TimeQuantum("YMDH") { + t.Fatalf("unexpected quantum: %#v", q) + } + }) + + t.Run("ErrInvalidTimeQuantum", func(t *testing.T) { + if _, err := pilosa.ParseTimeQuantum("BADQUANTUM"); err != pilosa.ErrInvalidTimeQuantum { + t.Fatalf("unexpected error: %s", err) + } + }) } -func TestTimeIDsFromRange_1h_1(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 10:03"), - *MustParseTime("2014-01-02 11:03"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } +// Ensure generated frame name can be returned for a given time unit. +func TestFrameByTimeUnit(t *testing.T) { + ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) + + t.Run("Y", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'Y'); s != "F_2000" { + t.Fatalf("unexpected name: %s", s) + } + }) + t.Run("M", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'M'); s != "F_200001" { + t.Fatalf("unexpected name: %s", s) + } + }) + t.Run("D", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'D'); s != "F_20000102" { + t.Fatalf("unexpected name: %s", s) + } + }) + t.Run("H", func(t *testing.T) { + if s := pilosa.FrameByTimeUnit("F", ts, 'H'); s != "F_2000010203" { + t.Fatalf("unexpected name: %s", s) + } + }) } -func TestTimeIDsFromRange_2h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 10:03"), - *MustParseTime("2014-01-02 12:03"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } +// Ensure all applicable frame names can be generated when mutating a time bit. +func TestFramesByTime(t *testing.T) { + ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) + + t.Run("YMDH", func(t *testing.T) { + a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("YMDH")) + if !reflect.DeepEqual(a, []string{"F_2000", "F_200001", "F_20000102", "F_2000010203"}) { + t.Fatalf("unexpected names: %+v", a) + } + }) + + t.Run("D", func(t *testing.T) { + a := pilosa.FramesByTime("F", ts, MustParseTimeQuantum("D")) + if !reflect.DeepEqual(a, []string{"F_20000102"}) { + t.Fatalf("unexpected names: %+v", a) + } + }) } -func TestTimeIDsFromRange_24h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 12:03"), - *MustParseTime("2014-01-03 12:03"), - uint64(1), - ); len(m) != 24 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1d(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 00:00"), - *MustParseTime("2014-01-03 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1d1h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 00:00"), - *MustParseTime("2014-01-03 01:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 23:00"), - *MustParseTime("2014-01-04 00:00"), - uint64(1), - ); len(m) != 2 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d1h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-02 23:00"), - *MustParseTime("2014-01-04 01:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1y(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-01 00:00"), - *MustParseTime("2015-01-01 00:00"), - uint64(1), - ); len(m) != 1 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d1m(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-30 23:00"), - *MustParseTime("2014-03-01 00:00"), - uint64(1), - ); len(m) != 3 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromRange_1h1d1m1d1h(t *testing.T) { - if m := pilosa.TimeIDsFromRange( - *MustParseTime("2014-01-30 23:00"), - *MustParseTime("2014-03-02 01:00"), - uint64(1), - ); len(m) != 5 { - t.Fatalf("unexpected range len: %d", len(m)) - } -} - -func TestTimeIDsFromQuantum(t *testing.T) { - _ = pilosa.TimeIDsFromQuantum(pilosa.YMD, *MustParseTime("1970-01-01 00:00"), uint64(15027)) +// Ensure sets of frames can be returned for a given time range. +func TestFramesByTimeRange(t *testing.T) { + t.Run("Y", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2002-01-01 00:00"), MustParseTimeQuantum("Y")) + if !reflect.DeepEqual(a, []string{"F_2000", "F_2001"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("YM", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-01 00:00"), MustParseTime("2003-03-01 00:00"), MustParseTimeQuantum("YM")) + if !reflect.DeepEqual(a, []string{"F_200011", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("YMD", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 00:00"), MustParseTime("2003-03-02 00:00"), MustParseTimeQuantum("YMD")) + if !reflect.DeepEqual(a, []string{"F_20001128", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_2002", "F_200301", "F_200302", "F_20030301"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("YMDH", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-28 22:00"), MustParseTime("2002-03-01 03:00"), MustParseTimeQuantum("YMDH")) + if !reflect.DeepEqual(a, []string{"F_2000112822", "F_2000112823", "F_20001129", "F_20001130", "F_200012", "F_2001", "F_200201", "F_200202", "F_2002030100", "F_2002030101", "F_2002030102"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("M", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-03-01 00:00"), MustParseTimeQuantum("M")) + if !reflect.DeepEqual(a, []string{"F_200001", "F_200002"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("MD", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 00:00"), MustParseTime("2002-02-03 00:00"), MustParseTimeQuantum("MD")) + if !reflect.DeepEqual(a, []string{"F_20001129", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_20020201", "F_20020202"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("MDH", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-11-29 22:00"), MustParseTime("2002-03-02 03:00"), MustParseTimeQuantum("MDH")) + if !reflect.DeepEqual(a, []string{"F_2000112922", "F_2000112923", "F_20001130", "F_200012", "F_200101", "F_200102", "F_200103", "F_200104", "F_200105", "F_200106", "F_200107", "F_200108", "F_200109", "F_200110", "F_200111", "F_200112", "F_200201", "F_200202", "F_20020301", "F_2002030200", "F_2002030201", "F_2002030202"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("D", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-04 00:00"), MustParseTimeQuantum("D")) + if !reflect.DeepEqual(a, []string{"F_20000101", "F_20000102", "F_20000103"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("DH", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 22:00"), MustParseTime("2000-03-01 02:00"), MustParseTimeQuantum("DH")) + if !reflect.DeepEqual(a, []string{"F_2000010122", "F_2000010123", "F_20000102", "F_20000103", "F_20000104", "F_20000105", "F_20000106", "F_20000107", "F_20000108", "F_20000109", "F_20000110", "F_20000111", "F_20000112", "F_20000113", "F_20000114", "F_20000115", "F_20000116", "F_20000117", "F_20000118", "F_20000119", "F_20000120", "F_20000121", "F_20000122", "F_20000123", "F_20000124", "F_20000125", "F_20000126", "F_20000127", "F_20000128", "F_20000129", "F_20000130", "F_20000131", "F_20000201", "F_20000202", "F_20000203", "F_20000204", "F_20000205", "F_20000206", "F_20000207", "F_20000208", "F_20000209", "F_20000210", "F_20000211", "F_20000212", "F_20000213", "F_20000214", "F_20000215", "F_20000216", "F_20000217", "F_20000218", "F_20000219", "F_20000220", "F_20000221", "F_20000222", "F_20000223", "F_20000224", "F_20000225", "F_20000226", "F_20000227", "F_20000228", "F_20000229", "F_2000030100", "F_2000030101"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) + t.Run("H", func(t *testing.T) { + a := pilosa.FramesByTimeRange("F", MustParseTime("2000-01-01 00:00"), MustParseTime("2000-01-01 02:00"), MustParseTimeQuantum("H")) + if !reflect.DeepEqual(a, []string{"F_2000010100", "F_2000010101"}) { + t.Fatalf("unexpected frames: %#v", a) + } + }) } // DefaultTimeLayout is the time layout used by the tests. const DefaultTimeLayout = "2006-01-02 15:04" // MustParseTime parses value using DefaultTimeLayout. Panic on error. -func MustParseTime(value string) *time.Time { +func MustParseTime(value string) time.Time { v, err := time.Parse(DefaultTimeLayout, value) if err != nil { panic(err) } + return v +} + +// MustParseTimePtr parses value using DefaultTimeLayout. Panic on error. +func MustParseTimePtr(value string) *time.Time { + v := MustParseTime(value) return &v } + +// MustParseTimeQuantum parses v into a time quantum. Panic on error. +func MustParseTimeQuantum(v string) pilosa.TimeQuantum { + q, err := pilosa.ParseTimeQuantum(v) + if err != nil { + panic(err) + } + return q +} From 9161ff122adcd17a77ed00b57d555eb806ec2c8a Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 13 Dec 2016 10:14:44 -0700 Subject: [PATCH 08/11] Fix frame SetBit slice calculation. --- frame.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frame.go b/frame.go index b215d80d2..684b142e7 100644 --- a/frame.go +++ b/frame.go @@ -288,7 +288,7 @@ func (f *Frame) newFragment(path string, slice uint64) *Fragment { // SetBit sets a bit within the frame. func (f *Frame) SetBit(bitmapID, profileID uint64) (changed bool, err error) { - slice := bitmapID / SliceWidth + slice := profileID / SliceWidth frag, err := f.CreateFragmentIfNotExists(slice) if err != nil { return changed, err From 7a91cbdd6d84c56d4fb0e93a077ae1e7f7c463f2 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 20 Dec 2016 10:31:24 -0700 Subject: [PATCH 09/11] Add timestamps to import/sort. The HTTP API, client, and `pilosactl` have been updated to allow for a timestamp formatted using the pilosa date format of `YYYY-MM-DDTHH:MM`. --- client.go | 21 ++- cmd/pilosactl/main.go | 56 +++++-- db.go | 90 ++++++++++ fragment.go | 2 +- handler.go | 26 ++- internal/internal.pb.go | 353 ++++++++++++++++++++++++++-------------- internal/internal.proto | 2 + pilosa.go | 3 + 8 files changed, 411 insertions(+), 142 deletions(-) diff --git a/client.go b/client.go index 47169571b..076bfa7f8 100644 --- a/client.go +++ b/client.go @@ -227,6 +227,7 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e // Separate bitmap and profile IDs to reduce allocations. bitmapIDs := Bits(bits).BitmapIDs() profileIDs := Bits(bits).ProfileIDs() + timestamps := Bits(bits).Timestamps() // Marshal bits to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ @@ -235,6 +236,7 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e Slice: slice, BitmapIDs: bitmapIDs, ProfileIDs: profileIDs, + Timestamps: timestamps, }) if err != nil { return nil, fmt.Errorf("marshal import request: %s", err) @@ -779,6 +781,7 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At type Bit struct { BitmapID uint64 ProfileID uint64 + Timestamp int64 } // Bits represents a slice of bits. @@ -789,6 +792,9 @@ func (p Bits) Len() int { return len(p) } func (p Bits) Less(i, j int) bool { if p[i].BitmapID == p[j].BitmapID { + if p[i].ProfileID < p[j].ProfileID { + return p[i].Timestamp < p[j].Timestamp + } return p[i].ProfileID < p[j].ProfileID } return p[i].BitmapID < p[j].BitmapID @@ -812,6 +818,15 @@ func (a Bits) ProfileIDs() []uint64 { return other } +// Timestamps returns a slice of all the timestamps. +func (a Bits) Timestamps() []int64 { + other := make([]int64, len(a)) + for i := range a { + other[i] = a[i].Timestamp + } + return other +} + // GroupBySlice returns a map of bits by slice. func (a Bits) GroupBySlice() map[uint64][]Bit { m := make(map[uint64][]Bit) @@ -834,5 +849,9 @@ type BitsByPos []Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitsByPos) Len() int { return len(p) } func (p BitsByPos) Less(i, j int) bool { - return Pos(p[i].BitmapID, p[i].ProfileID) < Pos(p[j].BitmapID, p[j].ProfileID) + p0, p1 := Pos(p[i].BitmapID, p[i].ProfileID), Pos(p[j].BitmapID, p[j].ProfileID) + if p0 == p1 { + return p[i].Timestamp < p[j].Timestamp + } + return p0 < p1 } diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 88b4c328f..0ec4f5600 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -270,9 +270,10 @@ of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: - BITMAPID,PROFILEID + BITMAPID,PROFILEID,[TIME] -The file should contain no headers. +The file should contain no headers. The TIME column is optional and can be +omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. `) } @@ -322,6 +323,7 @@ func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { // Read rows as bits. r := csv.NewReader(f) + r.FieldsPerRecord = -1 rnum := 0 for { rnum++ @@ -341,19 +343,32 @@ func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record)) } + var bit pilosa.Bit + // Parse bitmap id. bitmapID, err := strconv.ParseUint(record[0], 10, 64) if err != nil { return fmt.Errorf("invalid bitmap id on row %d: %q", rnum, record[0]) } + bit.BitmapID = bitmapID // Parse bitmap id. profileID, err := strconv.ParseUint(record[1], 10, 64) if err != nil { return fmt.Errorf("invalid profile id on row %d: %q", rnum, record[1]) } + bit.ProfileID = profileID - a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID}) + // Parse time, if exists. + if len(record) > 2 && record[2] != "" { + t, err := time.Parse(pilosa.TimeFormat, record[2]) + if err != nil { + return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2]) + } + bit.Timestamp = t.UnixNano() + } + + a = append(a, bit) // If we've reached the buffer size then import bits. if len(a) == cmd.BufferSize { @@ -568,9 +583,10 @@ func (cmd *SortCommand) Run(ctx context.Context) error { // Read rows as bits. r := csv.NewReader(f) + r.FieldsPerRecord = -1 a := make([]pilosa.Bit, 0, 1000000) for { - bitmapID, profileID, err := readCSVRow(r) + bitmapID, profileID, timestamp, err := readCSVRow(r) if err == io.EOF { break } else if err == errBlank { @@ -578,7 +594,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error { } else if err != nil { return err } - a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID}) + a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp}) } // Sort bits by position. @@ -591,8 +607,15 @@ func (cmd *SortCommand) Run(ctx context.Context) error { // Write CSV to buffer. buf = buf[:0] buf = strconv.AppendUint(buf, bit.BitmapID, 10) + buf = append(buf, ',') buf = strconv.AppendUint(buf, bit.ProfileID, 10) + + if bit.Timestamp != 0 { + buf = append(buf, ',') + buf = append(buf, time.Unix(0, bit.Timestamp).UTC().Format(pilosa.TimeFormat)...) + } + buf = append(buf, '\n') // Write to output. @@ -1148,33 +1171,42 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e } // readCSVRow reads a bitmap/profile pair from a CSV row. -func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, err error) { +func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) { // Read CSV row. record, err := r.Read() if err != nil { - return 0, 0, err + return 0, 0, 0, err } // Ignore blank rows. if record[0] == "" { - return 0, 0, errBlank + return 0, 0, 0, errBlank } else if len(record) < 2 { - return 0, 0, fmt.Errorf("bad column count: %d", len(record)) + return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record)) } // Parse bitmap id. bitmapID, err = strconv.ParseUint(record[0], 10, 64) if err != nil { - return 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0]) + return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0]) } // Parse bitmap id. profileID, err = strconv.ParseUint(record[1], 10, 64) if err != nil { - return 0, 0, fmt.Errorf("invalid profile id: %q", record[1]) + return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1]) } - return bitmapID, profileID, nil + // Parse timestamp, if available. + if len(record) > 2 && record[2] != "" { + t, err := time.Parse(pilosa.TimeFormat, record[2]) + if err != nil { + return 0, 0, 0, fmt.Errorf("invalid timestamp: %q", record[2]) + } + timestamp = t.UnixNano() + } + + return bitmapID, profileID, timestamp, nil } // errBlank indicates a blank row in a CSV file. diff --git a/db.go b/db.go index 6178cfc95..f2a5740bc 100644 --- a/db.go +++ b/db.go @@ -304,6 +304,15 @@ func (db *DB) DeleteFrame(name string) error { return nil } +// CreateFragmentIfNotExists returns a fragment in the database by name/slice. +func (db *DB) CreateFragmentIfNotExists(name string, slice uint64) (*Fragment, error) { + f, err := db.CreateFrameIfNotExists(name) + if err != nil { + return nil, err + } + return f.CreateFragmentIfNotExists(slice) +} + // SetBit sets a bit for a given profile & bitmap. // If a timestamp is specified then set all bits for the different quantum units. func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) { @@ -343,6 +352,67 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha return changed, nil } +// Import bulk imports data. +func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error { + // Read frame. + f, err := db.CreateFrameIfNotExists(name) + if err != nil { + return err + } + + // Determine quantum if timestamps are set. + var q TimeQuantum + if hasTime(timestamps) { + if q = f.TimeQuantum(); q == "" { + q = db.TimeQuantum() + if err := f.SetTimeQuantum(q); err != nil { + return err + } + } + + if q == "" { + return errors.New("time quantum not set in either database or frame") + } + } + + // Split import data by fragment. + dataByFragment := make(map[importKey]importData) + for i := range bitmapIDs { + bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i] + slice := profileID / SliceWidth + + var names []string + if timestamp == nil { + names = []string{name} + } else { + names = FramesByTime(name, *timestamp, q) + } + + // Attach bit to each frame. + for _, name := range names { + key := importKey{Frame: name, Slice: slice} + data := dataByFragment[key] + data.BitmapIDs = append(data.BitmapIDs, bitmapID) + data.ProfileIDs = append(data.ProfileIDs, profileID) + dataByFragment[key] = data + } + } + + // Import into each fragment. + for key, data := range dataByFragment { + f, err := db.CreateFragmentIfNotExists(key.Frame, key.Slice) + if err != nil { + return err + } + + if err := f.Import(data.BitmapIDs, data.ProfileIDs); err != nil { + return err + } + } + + return nil +} + type dbSlice []*DB func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -396,3 +466,23 @@ func (db *DB) SetRemoteMaxSlice(newmax uint64) { defer db.mu.Unlock() db.remoteMaxSlice = newmax } + +// hasTime returns true if a contains a non-nil time. +func hasTime(a []*time.Time) bool { + for _, t := range a { + if t != nil { + return true + } + } + return false +} + +type importKey struct { + Frame string + Slice uint64 +} + +type importData struct { + BitmapIDs []uint64 + ProfileIDs []uint64 +} diff --git a/fragment.go b/fragment.go index 3eea7cf03..60b2beec6 100644 --- a/fragment.go +++ b/fragment.go @@ -830,7 +830,7 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // Verify that there are an equal number of bitmap ids and profile ids. if len(bitmapIDs) != len(profileIDs) { - return fmt.Errorf("mismatch of bitmap and profile len: %d != %d", len(bitmapIDs), len(profileIDs)) + return fmt.Errorf("mismatch of bitmap/profile len: %d != %d", len(bitmapIDs), len(profileIDs)) } // Disconnect op writer so we don't append updates. diff --git a/handler.go b/handler.go index 1445d8c7c..2f08dc357 100644 --- a/handler.go +++ b/handler.go @@ -705,29 +705,37 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - db, frame, slice := req.DB, req.Frame, req.Slice + + // Convert timestamps to time.Time. + timestamps := make([]*time.Time, len(req.Timestamps)) + for i, ts := range req.Timestamps { + if ts == 0 { + continue + } + t := time.Unix(0, ts) + timestamps[i] = &t + } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Host, db, slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, db, slice) + if !h.Cluster.OwnsFragment(h.Host, req.DB, req.Slice) { + mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, req.DB, req.Slice) http.Error(w, mesg, http.StatusPreconditionFailed) return } // Find the correct fragment. - h.logger().Println("importing:", db, frame, slice) - f, err := h.Index.CreateFragmentIfNotExists(db, frame, slice) + h.logger().Println("importing:", req.DB, req.Frame, req.Slice) + db, err := h.Index.CreateDBIfNotExists(req.DB) if err != nil { - h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", db, frame, slice, err) + h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, err) http.Error(w, "fragment error", http.StatusInternalServerError) return } - h.logger().Println("Import into Fragment:", db, frame, slice, len(req.ProfileIDs)) // Import into fragment. - err = f.Import(req.BitmapIDs, req.ProfileIDs) + err = db.Import(req.Frame, req.BitmapIDs, req.ProfileIDs, timestamps) if err != nil { - h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", db, frame, slice, len(req.ProfileIDs), err) + h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err) } // Marshal response object. diff --git a/internal/internal.pb.go b/internal/internal.pb.go index 42edc2d46..20796686b 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -47,7 +47,7 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type DB struct { - TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,json=timeQuantum,proto3" json:"TimeQuantum,omitempty"` + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *DB) Reset() { *m = DB{} } @@ -56,7 +56,7 @@ func (*DB) ProtoMessage() {} func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } type Frame struct { - TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,json=timeQuantum,proto3" json:"TimeQuantum,omitempty"` + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *Frame) Reset() { *m = Frame{} } @@ -65,8 +65,8 @@ func (*Frame) ProtoMessage() {} func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } type Bitmap struct { - Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits,json=bits" json:"Bits,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` + Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Bitmap) Reset() { *m = Bitmap{} } @@ -82,8 +82,8 @@ func (m *Bitmap) GetAttrs() []*Attr { } type Pair struct { - Key uint64 `protobuf:"varint,1,opt,name=Key,json=key,proto3" json:"Key,omitempty"` - Count uint64 `protobuf:"varint,2,opt,name=Count,json=count,proto3" json:"Count,omitempty"` + Key uint64 `protobuf:"varint,1,opt,name=Key,proto3" json:"Key,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } func (m *Pair) Reset() { *m = Pair{} } @@ -92,8 +92,9 @@ func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } type Bit struct { - BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,json=bitmapID,proto3" json:"BitmapID,omitempty"` - ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,json=profileID,proto3" json:"ProfileID,omitempty"` + BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,proto3" json:"BitmapID,omitempty"` + ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,proto3" json:"ProfileID,omitempty"` + Timestamp int64 `protobuf:"varint,3,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` } func (m *Bit) Reset() { *m = Bit{} } @@ -102,8 +103,8 @@ func (*Bit) ProtoMessage() {} func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } type Profile struct { - ID uint64 `protobuf:"varint,1,opt,name=ID,json=iD,proto3" json:"ID,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Profile) Reset() { *m = Profile{} } @@ -119,11 +120,11 @@ func (m *Profile) GetAttrs() []*Attr { } type Attr struct { - Key string `protobuf:"bytes,1,opt,name=Key,json=key,proto3" json:"Key,omitempty"` - Type uint64 `protobuf:"varint,2,opt,name=Type,json=type,proto3" json:"Type,omitempty"` - StringValue string `protobuf:"bytes,3,opt,name=StringValue,json=stringValue,proto3" json:"StringValue,omitempty"` - UintValue uint64 `protobuf:"varint,4,opt,name=UintValue,json=uintValue,proto3" json:"UintValue,omitempty"` - BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,json=boolValue,proto3" json:"BoolValue,omitempty"` + Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` + Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` + StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` + UintValue uint64 `protobuf:"varint,4,opt,name=UintValue,proto3" json:"UintValue,omitempty"` + BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` } func (m *Attr) Reset() { *m = Attr{} } @@ -132,7 +133,7 @@ func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *AttrMap) Reset() { *m = AttrMap{} } @@ -148,13 +149,13 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - DB string `protobuf:"bytes,1,opt,name=DB,json=dB,proto3" json:"DB,omitempty"` - Query string `protobuf:"bytes,2,opt,name=Query,json=query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,3,rep,packed,name=Slices,json=slices" json:"Slices,omitempty"` - Profiles bool `protobuf:"varint,4,opt,name=Profiles,json=profiles,proto3" json:"Profiles,omitempty"` - Timestamp int64 `protobuf:"varint,5,opt,name=Timestamp,json=timestamp,proto3" json:"Timestamp,omitempty"` - Quantum string `protobuf:"bytes,6,opt,name=Quantum,json=quantum,proto3" json:"Quantum,omitempty"` - Remote bool `protobuf:"varint,7,opt,name=Remote,json=remote,proto3" json:"Remote,omitempty"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Query string `protobuf:"bytes,2,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,3,rep,packed,name=Slices" json:"Slices,omitempty"` + Profiles bool `protobuf:"varint,4,opt,name=Profiles,proto3" json:"Profiles,omitempty"` + Timestamp int64 `protobuf:"varint,5,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` + Quantum string `protobuf:"bytes,6,opt,name=Quantum,proto3" json:"Quantum,omitempty"` + Remote bool `protobuf:"varint,7,opt,name=Remote,proto3" json:"Remote,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } @@ -163,9 +164,9 @@ func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } type QueryResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,json=err,proto3" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results,json=results" json:"Results,omitempty"` - Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles,json=profiles" json:"Profiles,omitempty"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` } func (m *QueryResponse) Reset() { *m = QueryResponse{} } @@ -188,10 +189,10 @@ func (m *QueryResponse) GetProfiles() []*Profile { } type QueryResult struct { - Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap,json=bitmap" json:"Bitmap,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,json=n,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,json=pairs" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,json=changed,proto3" json:"Changed,omitempty"` + Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } @@ -214,11 +215,12 @@ func (m *QueryResult) GetPairs() []*Pair { } type ImportRequest struct { - DB string `protobuf:"bytes,1,opt,name=DB,json=dB,proto3" json:"DB,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,json=frame,proto3" json:"Frame,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,json=slice,proto3" json:"Slice,omitempty"` - BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs,json=profileIDs" json:"ProfileIDs,omitempty"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` + Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` } func (m *ImportRequest) Reset() { *m = ImportRequest{} } @@ -227,7 +229,7 @@ func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } type ImportResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,json=err,proto3" json:"Err,omitempty"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` } func (m *ImportResponse) Reset() { *m = ImportResponse{} } @@ -236,10 +238,10 @@ func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } type BlockDataRequest struct { - DB string `protobuf:"bytes,1,opt,name=DB,json=dB,proto3" json:"DB,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,json=frame,proto3" json:"Frame,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,json=slice,proto3" json:"Slice,omitempty"` - Block uint64 `protobuf:"varint,4,opt,name=Block,json=block,proto3" json:"Block,omitempty"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` } func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } @@ -248,8 +250,8 @@ func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } type BlockDataResponse struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs,json=profileIDs" json:"ProfileIDs,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` } func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } @@ -258,7 +260,7 @@ func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } type Cache struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` } func (m *Cache) Reset() { *m = Cache{} } @@ -267,7 +269,7 @@ func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{15} } type MaxSlicesResponse struct { - MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices,json=maxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } func (m *MaxSlicesResponse) Reset() { *m = MaxSlicesResponse{} } @@ -449,6 +451,11 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintInternal(dAtA, i, uint64(m.ProfileID)) } + if m.Timestamp != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Timestamp)) + } return i, nil } @@ -814,6 +821,24 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintInternal(dAtA, i, uint64(j8)) i += copy(dAtA[i:], dAtA9[:j8]) } + if len(m.Timestamps) > 0 { + dAtA11 := make([]byte, len(m.Timestamps)*10) + var j10 int + for _, num1 := range m.Timestamps { + num := uint64(num1) + for num >= 1<<7 { + dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j10++ + } + dAtA11[j10] = uint8(num) + j10++ + } + dAtA[i] = 0x32 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j10)) + i += copy(dAtA[i:], dAtA11[:j10]) + } return i, nil } @@ -897,26 +922,9 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.BitmapIDs) > 0 { - dAtA11 := make([]byte, len(m.BitmapIDs)*10) - var j10 int - for _, num := range m.BitmapIDs { - for num >= 1<<7 { - dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j10++ - } - dAtA11[j10] = uint8(num) - j10++ - } - dAtA[i] = 0xa - i++ - i = encodeVarintInternal(dAtA, i, uint64(j10)) - i += copy(dAtA[i:], dAtA11[:j10]) - } - if len(m.ProfileIDs) > 0 { - dAtA13 := make([]byte, len(m.ProfileIDs)*10) + dAtA13 := make([]byte, len(m.BitmapIDs)*10) var j12 int - for _, num := range m.ProfileIDs { + for _, num := range m.BitmapIDs { for num >= 1<<7 { dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -925,11 +933,28 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { dAtA13[j12] = uint8(num) j12++ } - dAtA[i] = 0x12 + dAtA[i] = 0xa i++ i = encodeVarintInternal(dAtA, i, uint64(j12)) i += copy(dAtA[i:], dAtA13[:j12]) } + if len(m.ProfileIDs) > 0 { + dAtA15 := make([]byte, len(m.ProfileIDs)*10) + var j14 int + for _, num := range m.ProfileIDs { + for num >= 1<<7 { + dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j14++ + } + dAtA15[j14] = uint8(num) + j14++ + } + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j14)) + i += copy(dAtA[i:], dAtA15[:j14]) + } return i, nil } @@ -949,21 +974,21 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.BitmapIDs) > 0 { - dAtA15 := make([]byte, len(m.BitmapIDs)*10) - var j14 int + dAtA17 := make([]byte, len(m.BitmapIDs)*10) + var j16 int for _, num := range m.BitmapIDs { for num >= 1<<7 { - dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) + dAtA17[j16] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j14++ + j16++ } - dAtA15[j14] = uint8(num) - j14++ + dAtA17[j16] = uint8(num) + j16++ } dAtA[i] = 0xa i++ - i = encodeVarintInternal(dAtA, i, uint64(j14)) - i += copy(dAtA[i:], dAtA15[:j14]) + i = encodeVarintInternal(dAtA, i, uint64(j16)) + i += copy(dAtA[i:], dAtA17[:j16]) } return i, nil } @@ -1089,6 +1114,9 @@ func (m *Bit) Size() (n int) { if m.ProfileID != 0 { n += 1 + sovInternal(uint64(m.ProfileID)) } + if m.Timestamp != 0 { + n += 1 + sovInternal(uint64(m.Timestamp)) + } return n } @@ -1248,6 +1276,13 @@ func (m *ImportRequest) Size() (n int) { } n += 1 + sovInternal(uint64(l)) + l } + if len(m.Timestamps) > 0 { + l = 0 + for _, e := range m.Timestamps { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } return n } @@ -1797,6 +1832,25 @@ func (m *Bit) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) + } + m.Timestamp = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Timestamp |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipInternal(dAtA[iNdEx:]) @@ -2947,6 +3001,68 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) } + case 6: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Timestamps = append(m.Timestamps, v) + } + } else if wireType == 0 { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Timestamps = append(m.Timestamps, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamps", wireType) + } default: iNdEx = preIndex skippy, err := skipInternal(dAtA[iNdEx:]) @@ -3743,51 +3859,50 @@ var ( func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 721 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xdd, 0x6a, 0x13, 0x41, - 0x14, 0x76, 0xf6, 0x27, 0x9b, 0x3d, 0x69, 0x43, 0x3a, 0x44, 0x59, 0x8a, 0x84, 0xb0, 0xa8, 0x44, - 0xc1, 0x16, 0xea, 0x8d, 0x88, 0x50, 0xdc, 0xa6, 0x62, 0x91, 0x4a, 0x3b, 0xad, 0xde, 0x6f, 0xd2, - 0x69, 0xbb, 0x74, 0xff, 0x3a, 0x33, 0x2b, 0xe6, 0xd2, 0x0b, 0x2f, 0xf4, 0x09, 0x04, 0x5f, 0xc4, - 0x47, 0xf0, 0xd2, 0x47, 0x90, 0xfa, 0x22, 0x32, 0xb3, 0x33, 0xd9, 0x94, 0x62, 0xed, 0x85, 0x97, - 0xdf, 0x77, 0x72, 0xe6, 0x7c, 0xe7, 0x7c, 0xe7, 0x6c, 0xa0, 0x9b, 0xe4, 0x82, 0xb2, 0x3c, 0x4e, - 0xd7, 0x4a, 0x56, 0x88, 0x02, 0xb7, 0x0d, 0x0e, 0x1f, 0x80, 0x35, 0x8e, 0xf0, 0x10, 0x3a, 0x87, - 0x49, 0x46, 0xf7, 0xab, 0x38, 0x17, 0x55, 0x16, 0xa0, 0x21, 0x1a, 0xf9, 0xa4, 0x23, 0x1a, 0x2a, - 0x7c, 0x08, 0xee, 0x4b, 0x16, 0x67, 0xf4, 0x06, 0x3f, 0x8d, 0xa0, 0x15, 0x25, 0x22, 0x8b, 0x4b, - 0x8c, 0xc1, 0x89, 0x12, 0xc1, 0x03, 0x34, 0xb4, 0x47, 0x0e, 0x71, 0x26, 0x89, 0xe0, 0xf8, 0x1e, - 0xb8, 0x2f, 0x84, 0x60, 0x3c, 0xb0, 0x86, 0xf6, 0xa8, 0xb3, 0xd1, 0x5d, 0x9b, 0x4b, 0x93, 0x34, - 0x71, 0x63, 0x19, 0x0c, 0xd7, 0xc0, 0xd9, 0x8b, 0x13, 0x86, 0x7b, 0x60, 0xbf, 0xa6, 0x33, 0x55, - 0xc5, 0x21, 0xf6, 0x19, 0x9d, 0xe1, 0x3e, 0xb8, 0x5b, 0x45, 0x95, 0x8b, 0xc0, 0x52, 0x9c, 0x3b, - 0x95, 0x20, 0xdc, 0x04, 0x3b, 0x4a, 0x04, 0x5e, 0x85, 0x76, 0x5d, 0x7a, 0x67, 0xac, 0x73, 0xda, - 0x13, 0x8d, 0xf1, 0x5d, 0xf0, 0xf7, 0x58, 0x71, 0x9c, 0xa4, 0x74, 0x67, 0xac, 0x93, 0xfd, 0xd2, - 0x10, 0xe1, 0x26, 0x78, 0x3a, 0x8a, 0xbb, 0x60, 0xcd, 0xd3, 0xad, 0x64, 0x7c, 0x43, 0xc5, 0x5f, - 0x10, 0x38, 0x12, 0x2f, 0x4a, 0xf6, 0x6b, 0xc9, 0x18, 0x9c, 0xc3, 0x59, 0x49, 0x75, 0x51, 0x47, - 0xcc, 0x4a, 0x35, 0xc6, 0x03, 0xc1, 0x92, 0xfc, 0xe4, 0x5d, 0x9c, 0x56, 0x34, 0xb0, 0xeb, 0x31, - 0xf2, 0x86, 0x92, 0x7a, 0xdf, 0x26, 0xb9, 0xa8, 0xe3, 0x4e, 0xad, 0xb7, 0x32, 0x84, 0x8c, 0x46, - 0x45, 0x91, 0xd6, 0x51, 0x77, 0x88, 0x46, 0x6d, 0xe2, 0x4f, 0x0c, 0x11, 0xae, 0x83, 0x27, 0xb5, - 0xec, 0xc6, 0x65, 0xa3, 0x1e, 0x5d, 0xa7, 0xfe, 0x3b, 0x82, 0xa5, 0xfd, 0x8a, 0xb2, 0x19, 0xa1, - 0xe7, 0x15, 0xe5, 0x42, 0x0e, 0x61, 0x1c, 0xe9, 0x26, 0xac, 0xa3, 0x48, 0x8e, 0x5d, 0xc5, 0x55, - 0x13, 0x3e, 0x71, 0xcf, 0x25, 0xc0, 0x77, 0xa0, 0x75, 0x90, 0x26, 0x53, 0xca, 0x03, 0x5b, 0x59, - 0xdc, 0xe2, 0x0a, 0x49, 0x1f, 0xf4, 0x34, 0xb9, 0x92, 0xde, 0x26, 0x6d, 0x3d, 0x6a, 0x2e, 0x95, - 0xcb, 0x05, 0xe2, 0x22, 0xce, 0x4a, 0xa5, 0xdc, 0x26, 0xbe, 0x30, 0x04, 0x0e, 0xc0, 0x33, 0xab, - 0xd5, 0x52, 0x95, 0xbc, 0xf3, 0x1a, 0xca, 0x5a, 0x84, 0x66, 0x85, 0xa0, 0x81, 0xa7, 0x5e, 0x6c, - 0x31, 0x85, 0xc2, 0x8f, 0x08, 0x96, 0xb5, 0x74, 0x5e, 0x16, 0x39, 0xa7, 0xd2, 0x81, 0x6d, 0xc6, - 0x8c, 0x03, 0x94, 0x31, 0xbc, 0x0e, 0x1e, 0xa1, 0xbc, 0x4a, 0x85, 0x31, 0xf1, 0x76, 0x33, 0x06, - 0x93, 0x5b, 0xa5, 0x82, 0x78, 0xac, 0xfe, 0x15, 0x7e, 0xbc, 0xd0, 0x80, 0xad, 0x32, 0x56, 0x9a, - 0x0c, 0x1d, 0x69, 0x7a, 0x0a, 0x3f, 0x21, 0xe8, 0x2c, 0xbc, 0x83, 0x47, 0xe6, 0x04, 0x94, 0x88, - 0xce, 0x46, 0xaf, 0x49, 0xae, 0x79, 0xd2, 0xaa, 0xf7, 0x12, 0x2f, 0x01, 0x7a, 0xa3, 0x17, 0x03, - 0xe5, 0xd2, 0x2c, 0xb9, 0xf6, 0xa6, 0xe6, 0x82, 0x59, 0x92, 0x26, 0x6e, 0x29, 0x83, 0x72, 0x46, - 0x5b, 0xa7, 0x71, 0x7e, 0x42, 0x8f, 0xf4, 0x70, 0xbd, 0x69, 0x0d, 0xc3, 0xcf, 0x08, 0x96, 0x77, - 0xb2, 0xb2, 0x60, 0xe2, 0x1a, 0x1f, 0xd5, 0x1d, 0x1b, 0x1f, 0x8f, 0xd5, 0x51, 0xf7, 0xc1, 0x55, - 0x3e, 0xaa, 0x3d, 0x74, 0x88, 0xab, 0x6c, 0x54, 0x3b, 0xa6, 0xaf, 0x47, 0xda, 0x28, 0x0d, 0xf6, - 0xcd, 0x39, 0x71, 0x3c, 0x00, 0x98, 0xdf, 0x13, 0x0f, 0x5c, 0x15, 0x86, 0xf9, 0x41, 0xf1, 0x30, - 0x84, 0xae, 0x91, 0xf2, 0x37, 0x5f, 0xc2, 0x23, 0xe8, 0x45, 0x69, 0x31, 0x3d, 0x1b, 0xc7, 0x22, - 0xfe, 0x1f, 0x8a, 0xfb, 0xe0, 0xaa, 0xf7, 0xf4, 0xbd, 0xb8, 0x13, 0x09, 0xc2, 0x7d, 0x58, 0x59, - 0xa8, 0xa2, 0xc5, 0x5c, 0x6a, 0x0e, 0x5d, 0xdf, 0x9c, 0x75, 0xa5, 0xb9, 0xfb, 0xe0, 0x6e, 0xc5, - 0xd3, 0xd3, 0x7f, 0x3c, 0x13, 0x7e, 0x43, 0xb0, 0xb2, 0x1b, 0x7f, 0xa8, 0x6f, 0x64, 0x5e, 0xfa, - 0x15, 0xf8, 0x73, 0x52, 0x9f, 0xe5, 0xa3, 0xc6, 0xe9, 0x2b, 0xbf, 0x6f, 0x98, 0xed, 0x5c, 0xb0, - 0x19, 0xf1, 0x33, 0x83, 0x57, 0x9f, 0x43, 0xf7, 0x72, 0x50, 0xce, 0xf8, 0xec, 0xf2, 0xd7, 0xa7, - 0x0f, 0xee, 0x7b, 0xf5, 0x95, 0xd0, 0x1f, 0x4c, 0x05, 0x9e, 0x59, 0x4f, 0x51, 0xd4, 0xfb, 0x71, - 0x31, 0x40, 0x3f, 0x2f, 0x06, 0xe8, 0xd7, 0xc5, 0x00, 0x7d, 0xfd, 0x3d, 0xb8, 0x35, 0x69, 0xa9, - 0xbf, 0x87, 0x27, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x8b, 0x4d, 0xcc, 0x03, 0x30, 0x06, 0x00, - 0x00, + // 706 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6a, 0x14, 0x4f, + 0x10, 0xff, 0xf7, 0xce, 0xcc, 0x7e, 0xd4, 0x26, 0xcb, 0xa6, 0xd9, 0xbf, 0x0c, 0x41, 0x96, 0xa1, + 0x51, 0x19, 0x05, 0x13, 0x88, 0x17, 0x11, 0x41, 0x9c, 0x6c, 0xc4, 0x45, 0x12, 0x92, 0x4e, 0xf4, + 0xe6, 0x61, 0x4c, 0xda, 0x64, 0xc8, 0x7c, 0xd9, 0xd3, 0x23, 0xee, 0xd1, 0x83, 0x17, 0x9f, 0x40, + 0xf0, 0x09, 0x7c, 0x03, 0x1f, 0xc1, 0xa3, 0x8f, 0x20, 0xf1, 0x45, 0xa4, 0x3f, 0xe6, 0x23, 0x11, + 0x63, 0x0e, 0xde, 0xa6, 0x7e, 0xd5, 0xd5, 0x55, 0xbf, 0xfa, 0x55, 0xf5, 0xc0, 0x28, 0x4a, 0x05, + 0xe3, 0x69, 0x18, 0xaf, 0xe5, 0x3c, 0x13, 0x19, 0xee, 0x57, 0x36, 0xb9, 0x05, 0x9d, 0x59, 0x80, + 0x3d, 0x18, 0x1e, 0x44, 0x09, 0xdb, 0x2b, 0xc3, 0x54, 0x94, 0x89, 0x8b, 0x3c, 0xe4, 0x0f, 0x68, + 0x1b, 0x22, 0xb7, 0xc1, 0x79, 0xc2, 0xc3, 0x84, 0x5d, 0xe1, 0x68, 0x00, 0xdd, 0x20, 0x12, 0x49, + 0x98, 0x63, 0x0c, 0x76, 0x10, 0x89, 0xc2, 0x45, 0x9e, 0xe5, 0xdb, 0x54, 0x7d, 0xe3, 0x1b, 0xe0, + 0x3c, 0x16, 0x82, 0x17, 0x6e, 0xc7, 0xb3, 0xfc, 0xe1, 0xc6, 0x68, 0xad, 0x2e, 0x4d, 0xc2, 0x54, + 0x3b, 0xc9, 0x1a, 0xd8, 0xbb, 0x61, 0xc4, 0xf1, 0x18, 0xac, 0x67, 0x6c, 0xa1, 0xb2, 0xd8, 0x54, + 0x7e, 0xe2, 0x09, 0x38, 0x9b, 0x59, 0x99, 0x0a, 0xb7, 0xa3, 0x30, 0x6d, 0x90, 0x97, 0x60, 0x05, + 0x91, 0xc0, 0xab, 0xd0, 0xd7, 0xa9, 0xe7, 0x33, 0x13, 0x53, 0xdb, 0xf8, 0x3a, 0x0c, 0x76, 0x79, + 0xf6, 0x3a, 0x8a, 0xd9, 0x7c, 0x66, 0x82, 0x1b, 0x40, 0x7a, 0x25, 0x87, 0x42, 0x84, 0x49, 0xee, + 0x5a, 0x1e, 0xf2, 0x2d, 0xda, 0x00, 0xe4, 0x11, 0xf4, 0xcc, 0x51, 0x3c, 0x82, 0x4e, 0x7d, 0x79, + 0x67, 0x3e, 0xbb, 0x22, 0x9f, 0x8f, 0x08, 0x6c, 0xf9, 0xd5, 0x26, 0x34, 0xd0, 0x84, 0x30, 0xd8, + 0x07, 0x8b, 0x9c, 0x99, 0x92, 0xd4, 0xb7, 0x6c, 0xf2, 0xbe, 0xe0, 0x51, 0x7a, 0xfc, 0x22, 0x8c, + 0x4b, 0xa6, 0xea, 0x19, 0xd0, 0x36, 0x24, 0xeb, 0x7d, 0x1e, 0xa5, 0x42, 0xfb, 0x6d, 0xcd, 0xa6, + 0x06, 0xa4, 0x37, 0xc8, 0xb2, 0x58, 0x7b, 0x1d, 0x0f, 0xf9, 0x7d, 0xda, 0x00, 0x64, 0x1d, 0x7a, + 0xb2, 0x96, 0xed, 0x30, 0x6f, 0xaa, 0x47, 0x97, 0x55, 0xff, 0x15, 0xc1, 0xd2, 0x5e, 0xc9, 0xf8, + 0x82, 0xb2, 0x37, 0x25, 0x2b, 0x84, 0x6c, 0xc2, 0x2c, 0x30, 0x24, 0xe4, 0xfc, 0x4c, 0xc0, 0x51, + 0x7e, 0x45, 0x62, 0x40, 0xb5, 0x81, 0xaf, 0x41, 0x77, 0x3f, 0x8e, 0x0e, 0x59, 0xe1, 0x5a, 0x6a, + 0x00, 0x8c, 0x25, 0x55, 0x32, 0xdd, 0x2c, 0x54, 0xe9, 0x7d, 0x5a, 0xdb, 0xe7, 0x75, 0x70, 0x2e, + 0xe8, 0x80, 0x5d, 0xe8, 0x55, 0x83, 0xd7, 0x55, 0x99, 0x2a, 0x53, 0xe6, 0xa2, 0x2c, 0xc9, 0x04, + 0x73, 0x7b, 0xea, 0x46, 0x63, 0x91, 0xf7, 0x08, 0x96, 0x4d, 0xe9, 0x45, 0x9e, 0xa5, 0x05, 0x93, + 0x0a, 0x6c, 0x71, 0x5e, 0x29, 0xb0, 0xc5, 0x39, 0x5e, 0x87, 0x1e, 0x65, 0x45, 0x19, 0x8b, 0x4a, + 0xc4, 0xff, 0x9b, 0x36, 0x54, 0xb1, 0x65, 0x2c, 0x68, 0x75, 0x0a, 0xdf, 0x6d, 0x11, 0xb0, 0x54, + 0xc4, 0x4a, 0x13, 0x61, 0x3c, 0x0d, 0x27, 0xf2, 0x01, 0xc1, 0xb0, 0x75, 0x0f, 0xf6, 0xab, 0x05, + 0x51, 0x45, 0x0c, 0x37, 0xc6, 0x4d, 0xb0, 0xc6, 0x69, 0xb5, 0x40, 0x4b, 0x80, 0x76, 0xcc, 0x60, + 0xa0, 0x1d, 0x29, 0x96, 0x5c, 0x8a, 0x2a, 0x67, 0x4b, 0x2c, 0x09, 0x53, 0xed, 0x94, 0x3d, 0xda, + 0x3c, 0x09, 0xd3, 0x63, 0x76, 0x64, 0x9a, 0x5b, 0x99, 0xe4, 0x0b, 0x82, 0xe5, 0x79, 0x92, 0x67, + 0x5c, 0x5c, 0xa2, 0xa3, 0xda, 0xf2, 0x4a, 0x47, 0xbd, 0xf2, 0x13, 0x70, 0x94, 0x72, 0x6a, 0x0e, + 0x6d, 0xaa, 0x0d, 0x35, 0x63, 0x66, 0xb7, 0xa4, 0x8c, 0x52, 0xe0, 0x06, 0xc0, 0x53, 0x80, 0x7a, + 0xb9, 0x0a, 0xd7, 0x51, 0xee, 0x16, 0x22, 0xfd, 0xb5, 0xac, 0x85, 0xdb, 0xf5, 0x2c, 0xdf, 0xa2, + 0x2d, 0x84, 0x10, 0x18, 0x55, 0xa5, 0xfe, 0x49, 0x37, 0x72, 0x04, 0xe3, 0x20, 0xce, 0x0e, 0x4f, + 0x67, 0xa1, 0x08, 0xff, 0x05, 0xa3, 0x09, 0x38, 0xea, 0x3e, 0xb3, 0x4f, 0xda, 0x20, 0x7b, 0xb0, + 0xd2, 0xca, 0x62, 0x8a, 0x39, 0x47, 0x1e, 0x5d, 0x4e, 0xbe, 0x73, 0x91, 0x3c, 0xb9, 0x09, 0xce, + 0x66, 0x78, 0x78, 0xf2, 0x97, 0x6b, 0xc8, 0x67, 0x04, 0x2b, 0xdb, 0xe1, 0x3b, 0xbd, 0x35, 0x75, + 0xea, 0xa7, 0x30, 0xa8, 0x41, 0xb3, 0xb6, 0x77, 0x9a, 0x49, 0xf8, 0xed, 0x7c, 0x83, 0x6c, 0xa5, + 0x82, 0x2f, 0x68, 0x13, 0xbc, 0xfa, 0x10, 0x46, 0xe7, 0x9d, 0xb2, 0xc7, 0xa7, 0xcd, 0xeb, 0x74, + 0xaa, 0x9f, 0xdb, 0xb7, 0xea, 0x15, 0x31, 0xcf, 0xad, 0x32, 0x1e, 0x74, 0xee, 0xa3, 0x60, 0xfc, + 0xed, 0x6c, 0x8a, 0xbe, 0x9f, 0x4d, 0xd1, 0x8f, 0xb3, 0x29, 0xfa, 0xf4, 0x73, 0xfa, 0xdf, 0xab, + 0xae, 0xfa, 0xb9, 0xdc, 0xfb, 0x15, 0x00, 0x00, 0xff, 0xff, 0xe9, 0x7d, 0x09, 0xe4, 0x6e, 0x06, + 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index a26cd1f02..bab66e6bf 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -23,6 +23,7 @@ message Pair { message Bit { uint64 BitmapID = 1; uint64 ProfileID = 2; + int64 Timestamp = 3; } message Profile { @@ -71,6 +72,7 @@ message ImportRequest { uint64 Slice = 3; repeated uint64 BitmapIDs = 4; repeated uint64 ProfileIDs = 5; + repeated int64 Timestamps = 6; } message ImportResponse { diff --git a/pilosa.go b/pilosa.go index d6c07bb0c..a93dc241c 100644 --- a/pilosa.go +++ b/pilosa.go @@ -75,3 +75,6 @@ func decodeProfile(pb *internal.Profile) *Profile { return p } + +// TimeFormat is the go-style time format used to parse string dates. +const TimeFormat = "2006-01-02T15:04" From 225bddc2a8f45a22f08aba8590da82309b2e9601 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Wed, 4 Jan 2017 15:03:55 -0600 Subject: [PATCH 10/11] Add handler endpoint to get list of cluster nodes --- handler.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/handler.go b/handler.go index 2f08dc357..175161384 100644 --- a/handler.go +++ b/handler.go @@ -186,6 +186,13 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) } + case "/nodes": + switch r.Method { + case "GET": + h.handleGetNodes(w, r) + default: + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + } case "/version": h.handleVersion(w, r) @@ -1012,6 +1019,13 @@ func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) } } +// handleGetNodes handles /nodes requests. +func (h *Handler) handleGetNodes(w http.ResponseWriter, r *http.Request) { + if err := json.NewEncoder(w).Encode(h.Cluster.Nodes); err != nil { + h.logger().Printf("write version response error: %s", err) + } +} + // handleGetVersion handles /version requests. func (h *Handler) handleVersion(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(struct { From ac6eb21a0fab0241e3dd558601159f1b46db4ef0 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Thu, 5 Jan 2017 14:23:51 -0600 Subject: [PATCH 11/11] Add github.com/golang/protobuf package to Glide Import was added in 3d0dada6fba4b039458eae4a0ea7633c1fb5ed93. --- glide.lock | 8 ++++++-- glide.yaml | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/glide.lock b/glide.lock index b5fcc4f26..e44487d48 100644 --- a/glide.lock +++ b/glide.lock @@ -1,5 +1,5 @@ -hash: 9afe91839785e89880061fd2919b0a5cdb10d6f4c06b344c2924eeed0c9bfd65 -updated: 2016-11-29T12:31:19.234235928-06:00 +hash: f60caa09b23585e33359d4ee206e0a1793633095cb92c7623609e9876ad697ab +updated: 2017-01-05T14:22:16.623903529-06:00 imports: - name: github.com/boltdb/bolt version: 4b1ebc1869ad66568b313d0dc410e2be72670dda @@ -21,6 +21,10 @@ imports: version: a6b377e3400b08991b80d6805d627f347f983866 subpackages: - lru +- name: github.com/golang/protobuf + version: 8ee79997227bf9b34611aee7946ae64735e6fd93 + subpackages: + - proto - name: golang.org/x/sys version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: diff --git a/glide.yaml b/glide.yaml index 55c2fc4e6..789076840 100644 --- a/glide.yaml +++ b/glide.yaml @@ -24,3 +24,4 @@ import: version: c200b10b5d5e122be351b67af224adc6128af5bf subpackages: - unix +- package: github.com/golang/protobuf