From 746760823406070e98567712e801351346432be2 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Tue, 20 Dec 2016 10:31:24 -0700 Subject: [PATCH] Add timestamps to import/sort. The HTTP API, client, and `pilosactl` have been updated to allow for a timestamp formatted using the pilosa date format of `YYYY-MM-DDTHH:MM`. --- client.go | 21 ++- cmd/pilosactl/main.go | 56 +++++-- db.go | 90 ++++++++++ fragment.go | 2 +- handler.go | 26 ++- internal/internal.pb.go | 353 ++++++++++++++++++++++++++-------------- internal/internal.proto | 2 + pilosa.go | 3 + 8 files changed, 411 insertions(+), 142 deletions(-) diff --git a/client.go b/client.go index 47169571b..076bfa7f8 100644 --- a/client.go +++ b/client.go @@ -227,6 +227,7 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e // Separate bitmap and profile IDs to reduce allocations. bitmapIDs := Bits(bits).BitmapIDs() profileIDs := Bits(bits).ProfileIDs() + timestamps := Bits(bits).Timestamps() // Marshal bits to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ @@ -235,6 +236,7 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e Slice: slice, BitmapIDs: bitmapIDs, ProfileIDs: profileIDs, + Timestamps: timestamps, }) if err != nil { return nil, fmt.Errorf("marshal import request: %s", err) @@ -779,6 +781,7 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At type Bit struct { BitmapID uint64 ProfileID uint64 + Timestamp int64 } // Bits represents a slice of bits. @@ -789,6 +792,9 @@ func (p Bits) Len() int { return len(p) } func (p Bits) Less(i, j int) bool { if p[i].BitmapID == p[j].BitmapID { + if p[i].ProfileID < p[j].ProfileID { + return p[i].Timestamp < p[j].Timestamp + } return p[i].ProfileID < p[j].ProfileID } return p[i].BitmapID < p[j].BitmapID @@ -812,6 +818,15 @@ func (a Bits) ProfileIDs() []uint64 { return other } +// Timestamps returns a slice of all the timestamps. +func (a Bits) Timestamps() []int64 { + other := make([]int64, len(a)) + for i := range a { + other[i] = a[i].Timestamp + } + return other +} + // GroupBySlice returns a map of bits by slice. func (a Bits) GroupBySlice() map[uint64][]Bit { m := make(map[uint64][]Bit) @@ -834,5 +849,9 @@ type BitsByPos []Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitsByPos) Len() int { return len(p) } func (p BitsByPos) Less(i, j int) bool { - return Pos(p[i].BitmapID, p[i].ProfileID) < Pos(p[j].BitmapID, p[j].ProfileID) + p0, p1 := Pos(p[i].BitmapID, p[i].ProfileID), Pos(p[j].BitmapID, p[j].ProfileID) + if p0 == p1 { + return p[i].Timestamp < p[j].Timestamp + } + return p0 < p1 } diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 88b4c328f..0ec4f5600 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -270,9 +270,10 @@ of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: - BITMAPID,PROFILEID + BITMAPID,PROFILEID,[TIME] -The file should contain no headers. +The file should contain no headers. The TIME column is optional and can be +omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. `) } @@ -322,6 +323,7 @@ func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { // Read rows as bits. r := csv.NewReader(f) + r.FieldsPerRecord = -1 rnum := 0 for { rnum++ @@ -341,19 +343,32 @@ func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record)) } + var bit pilosa.Bit + // Parse bitmap id. bitmapID, err := strconv.ParseUint(record[0], 10, 64) if err != nil { return fmt.Errorf("invalid bitmap id on row %d: %q", rnum, record[0]) } + bit.BitmapID = bitmapID // Parse bitmap id. profileID, err := strconv.ParseUint(record[1], 10, 64) if err != nil { return fmt.Errorf("invalid profile id on row %d: %q", rnum, record[1]) } + bit.ProfileID = profileID - a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID}) + // Parse time, if exists. + if len(record) > 2 && record[2] != "" { + t, err := time.Parse(pilosa.TimeFormat, record[2]) + if err != nil { + return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2]) + } + bit.Timestamp = t.UnixNano() + } + + a = append(a, bit) // If we've reached the buffer size then import bits. if len(a) == cmd.BufferSize { @@ -568,9 +583,10 @@ func (cmd *SortCommand) Run(ctx context.Context) error { // Read rows as bits. r := csv.NewReader(f) + r.FieldsPerRecord = -1 a := make([]pilosa.Bit, 0, 1000000) for { - bitmapID, profileID, err := readCSVRow(r) + bitmapID, profileID, timestamp, err := readCSVRow(r) if err == io.EOF { break } else if err == errBlank { @@ -578,7 +594,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error { } else if err != nil { return err } - a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID}) + a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp}) } // Sort bits by position. @@ -591,8 +607,15 @@ func (cmd *SortCommand) Run(ctx context.Context) error { // Write CSV to buffer. buf = buf[:0] buf = strconv.AppendUint(buf, bit.BitmapID, 10) + buf = append(buf, ',') buf = strconv.AppendUint(buf, bit.ProfileID, 10) + + if bit.Timestamp != 0 { + buf = append(buf, ',') + buf = append(buf, time.Unix(0, bit.Timestamp).UTC().Format(pilosa.TimeFormat)...) + } + buf = append(buf, '\n') // Write to output. @@ -1148,33 +1171,42 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e } // readCSVRow reads a bitmap/profile pair from a CSV row. -func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, err error) { +func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) { // Read CSV row. record, err := r.Read() if err != nil { - return 0, 0, err + return 0, 0, 0, err } // Ignore blank rows. if record[0] == "" { - return 0, 0, errBlank + return 0, 0, 0, errBlank } else if len(record) < 2 { - return 0, 0, fmt.Errorf("bad column count: %d", len(record)) + return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record)) } // Parse bitmap id. bitmapID, err = strconv.ParseUint(record[0], 10, 64) if err != nil { - return 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0]) + return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0]) } // Parse bitmap id. profileID, err = strconv.ParseUint(record[1], 10, 64) if err != nil { - return 0, 0, fmt.Errorf("invalid profile id: %q", record[1]) + return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1]) } - return bitmapID, profileID, nil + // Parse timestamp, if available. + if len(record) > 2 && record[2] != "" { + t, err := time.Parse(pilosa.TimeFormat, record[2]) + if err != nil { + return 0, 0, 0, fmt.Errorf("invalid timestamp: %q", record[2]) + } + timestamp = t.UnixNano() + } + + return bitmapID, profileID, timestamp, nil } // errBlank indicates a blank row in a CSV file. diff --git a/db.go b/db.go index 6178cfc95..f2a5740bc 100644 --- a/db.go +++ b/db.go @@ -304,6 +304,15 @@ func (db *DB) DeleteFrame(name string) error { return nil } +// CreateFragmentIfNotExists returns a fragment in the database by name/slice. +func (db *DB) CreateFragmentIfNotExists(name string, slice uint64) (*Fragment, error) { + f, err := db.CreateFrameIfNotExists(name) + if err != nil { + return nil, err + } + return f.CreateFragmentIfNotExists(slice) +} + // SetBit sets a bit for a given profile & bitmap. // If a timestamp is specified then set all bits for the different quantum units. func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (changed bool, err error) { @@ -343,6 +352,67 @@ func (db *DB) SetBit(name string, bitmapID, profileID uint64, t *time.Time) (cha return changed, nil } +// Import bulk imports data. +func (db *DB) Import(name string, bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error { + // Read frame. + f, err := db.CreateFrameIfNotExists(name) + if err != nil { + return err + } + + // Determine quantum if timestamps are set. + var q TimeQuantum + if hasTime(timestamps) { + if q = f.TimeQuantum(); q == "" { + q = db.TimeQuantum() + if err := f.SetTimeQuantum(q); err != nil { + return err + } + } + + if q == "" { + return errors.New("time quantum not set in either database or frame") + } + } + + // Split import data by fragment. + dataByFragment := make(map[importKey]importData) + for i := range bitmapIDs { + bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i] + slice := profileID / SliceWidth + + var names []string + if timestamp == nil { + names = []string{name} + } else { + names = FramesByTime(name, *timestamp, q) + } + + // Attach bit to each frame. + for _, name := range names { + key := importKey{Frame: name, Slice: slice} + data := dataByFragment[key] + data.BitmapIDs = append(data.BitmapIDs, bitmapID) + data.ProfileIDs = append(data.ProfileIDs, profileID) + dataByFragment[key] = data + } + } + + // Import into each fragment. + for key, data := range dataByFragment { + f, err := db.CreateFragmentIfNotExists(key.Frame, key.Slice) + if err != nil { + return err + } + + if err := f.Import(data.BitmapIDs, data.ProfileIDs); err != nil { + return err + } + } + + return nil +} + type dbSlice []*DB func (p dbSlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -396,3 +466,23 @@ func (db *DB) SetRemoteMaxSlice(newmax uint64) { defer db.mu.Unlock() db.remoteMaxSlice = newmax } + +// hasTime returns true if a contains a non-nil time. +func hasTime(a []*time.Time) bool { + for _, t := range a { + if t != nil { + return true + } + } + return false +} + +type importKey struct { + Frame string + Slice uint64 +} + +type importData struct { + BitmapIDs []uint64 + ProfileIDs []uint64 +} diff --git a/fragment.go b/fragment.go index 3eea7cf03..60b2beec6 100644 --- a/fragment.go +++ b/fragment.go @@ -830,7 +830,7 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // Verify that there are an equal number of bitmap ids and profile ids. if len(bitmapIDs) != len(profileIDs) { - return fmt.Errorf("mismatch of bitmap and profile len: %d != %d", len(bitmapIDs), len(profileIDs)) + return fmt.Errorf("mismatch of bitmap/profile len: %d != %d", len(bitmapIDs), len(profileIDs)) } // Disconnect op writer so we don't append updates. diff --git a/handler.go b/handler.go index 1445d8c7c..2f08dc357 100644 --- a/handler.go +++ b/handler.go @@ -705,29 +705,37 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { http.Error(w, err.Error(), http.StatusBadRequest) return } - db, frame, slice := req.DB, req.Frame, req.Slice + + // Convert timestamps to time.Time. + timestamps := make([]*time.Time, len(req.Timestamps)) + for i, ts := range req.Timestamps { + if ts == 0 { + continue + } + t := time.Unix(0, ts) + timestamps[i] = &t + } // Validate that this handler owns the slice. - if !h.Cluster.OwnsFragment(h.Host, db, slice) { - mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, db, slice) + if !h.Cluster.OwnsFragment(h.Host, req.DB, req.Slice) { + mesg := fmt.Sprintf("host does not own slice %s-%s slice:%d", h.Host, req.DB, req.Slice) http.Error(w, mesg, http.StatusPreconditionFailed) return } // Find the correct fragment. - h.logger().Println("importing:", db, frame, slice) - f, err := h.Index.CreateFragmentIfNotExists(db, frame, slice) + h.logger().Println("importing:", req.DB, req.Frame, req.Slice) + db, err := h.Index.CreateDBIfNotExists(req.DB) if err != nil { - h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", db, frame, slice, err) + h.logger().Printf("fragment error: db=%s, frame=%s, slice=%d, err=%s", req.DB, req.Frame, req.Slice, err) http.Error(w, "fragment error", http.StatusInternalServerError) return } - h.logger().Println("Import into Fragment:", db, frame, slice, len(req.ProfileIDs)) // Import into fragment. - err = f.Import(req.BitmapIDs, req.ProfileIDs) + err = db.Import(req.Frame, req.BitmapIDs, req.ProfileIDs, timestamps) if err != nil { - h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", db, frame, slice, len(req.ProfileIDs), err) + h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err) } // Marshal response object. diff --git a/internal/internal.pb.go b/internal/internal.pb.go index 42edc2d46..20796686b 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -47,7 +47,7 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type DB struct { - TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,json=timeQuantum,proto3" json:"TimeQuantum,omitempty"` + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *DB) Reset() { *m = DB{} } @@ -56,7 +56,7 @@ func (*DB) ProtoMessage() {} func (*DB) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{0} } type Frame struct { - TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,json=timeQuantum,proto3" json:"TimeQuantum,omitempty"` + TimeQuantum string `protobuf:"bytes,1,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` } func (m *Frame) Reset() { *m = Frame{} } @@ -65,8 +65,8 @@ func (*Frame) ProtoMessage() {} func (*Frame) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{1} } type Bitmap struct { - Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits,json=bits" json:"Bits,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` + Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Bitmap) Reset() { *m = Bitmap{} } @@ -82,8 +82,8 @@ func (m *Bitmap) GetAttrs() []*Attr { } type Pair struct { - Key uint64 `protobuf:"varint,1,opt,name=Key,json=key,proto3" json:"Key,omitempty"` - Count uint64 `protobuf:"varint,2,opt,name=Count,json=count,proto3" json:"Count,omitempty"` + Key uint64 `protobuf:"varint,1,opt,name=Key,proto3" json:"Key,omitempty"` + Count uint64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"` } func (m *Pair) Reset() { *m = Pair{} } @@ -92,8 +92,9 @@ func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{3} } type Bit struct { - BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,json=bitmapID,proto3" json:"BitmapID,omitempty"` - ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,json=profileID,proto3" json:"ProfileID,omitempty"` + BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,proto3" json:"BitmapID,omitempty"` + ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,proto3" json:"ProfileID,omitempty"` + Timestamp int64 `protobuf:"varint,3,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` } func (m *Bit) Reset() { *m = Bit{} } @@ -102,8 +103,8 @@ func (*Bit) ProtoMessage() {} func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{4} } type Profile struct { - ID uint64 `protobuf:"varint,1,opt,name=ID,json=iD,proto3" json:"ID,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` + ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Profile) Reset() { *m = Profile{} } @@ -119,11 +120,11 @@ func (m *Profile) GetAttrs() []*Attr { } type Attr struct { - Key string `protobuf:"bytes,1,opt,name=Key,json=key,proto3" json:"Key,omitempty"` - Type uint64 `protobuf:"varint,2,opt,name=Type,json=type,proto3" json:"Type,omitempty"` - StringValue string `protobuf:"bytes,3,opt,name=StringValue,json=stringValue,proto3" json:"StringValue,omitempty"` - UintValue uint64 `protobuf:"varint,4,opt,name=UintValue,json=uintValue,proto3" json:"UintValue,omitempty"` - BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,json=boolValue,proto3" json:"BoolValue,omitempty"` + Key string `protobuf:"bytes,1,opt,name=Key,proto3" json:"Key,omitempty"` + Type uint64 `protobuf:"varint,2,opt,name=Type,proto3" json:"Type,omitempty"` + StringValue string `protobuf:"bytes,3,opt,name=StringValue,proto3" json:"StringValue,omitempty"` + UintValue uint64 `protobuf:"varint,4,opt,name=UintValue,proto3" json:"UintValue,omitempty"` + BoolValue bool `protobuf:"varint,5,opt,name=BoolValue,proto3" json:"BoolValue,omitempty"` } func (m *Attr) Reset() { *m = Attr{} } @@ -132,7 +133,7 @@ func (*Attr) ProtoMessage() {} func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{6} } type AttrMap struct { - Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs,json=attrs" json:"Attrs,omitempty"` + Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *AttrMap) Reset() { *m = AttrMap{} } @@ -148,13 +149,13 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - DB string `protobuf:"bytes,1,opt,name=DB,json=dB,proto3" json:"DB,omitempty"` - Query string `protobuf:"bytes,2,opt,name=Query,json=query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,3,rep,packed,name=Slices,json=slices" json:"Slices,omitempty"` - Profiles bool `protobuf:"varint,4,opt,name=Profiles,json=profiles,proto3" json:"Profiles,omitempty"` - Timestamp int64 `protobuf:"varint,5,opt,name=Timestamp,json=timestamp,proto3" json:"Timestamp,omitempty"` - Quantum string `protobuf:"bytes,6,opt,name=Quantum,json=quantum,proto3" json:"Quantum,omitempty"` - Remote bool `protobuf:"varint,7,opt,name=Remote,json=remote,proto3" json:"Remote,omitempty"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Query string `protobuf:"bytes,2,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,3,rep,packed,name=Slices" json:"Slices,omitempty"` + Profiles bool `protobuf:"varint,4,opt,name=Profiles,proto3" json:"Profiles,omitempty"` + Timestamp int64 `protobuf:"varint,5,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` + Quantum string `protobuf:"bytes,6,opt,name=Quantum,proto3" json:"Quantum,omitempty"` + Remote bool `protobuf:"varint,7,opt,name=Remote,proto3" json:"Remote,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } @@ -163,9 +164,9 @@ func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{8} } type QueryResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,json=err,proto3" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results,json=results" json:"Results,omitempty"` - Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles,json=profiles" json:"Profiles,omitempty"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` } func (m *QueryResponse) Reset() { *m = QueryResponse{} } @@ -188,10 +189,10 @@ func (m *QueryResponse) GetProfiles() []*Profile { } type QueryResult struct { - Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap,json=bitmap" json:"Bitmap,omitempty"` - N uint64 `protobuf:"varint,2,opt,name=N,json=n,proto3" json:"N,omitempty"` - Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs,json=pairs" json:"Pairs,omitempty"` - Changed bool `protobuf:"varint,4,opt,name=Changed,json=changed,proto3" json:"Changed,omitempty"` + Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"` + N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"` + Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"` + Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"` } func (m *QueryResult) Reset() { *m = QueryResult{} } @@ -214,11 +215,12 @@ func (m *QueryResult) GetPairs() []*Pair { } type ImportRequest struct { - DB string `protobuf:"bytes,1,opt,name=DB,json=dB,proto3" json:"DB,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,json=frame,proto3" json:"Frame,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,json=slice,proto3" json:"Slice,omitempty"` - BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs,json=profileIDs" json:"ProfileIDs,omitempty"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` + Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` } func (m *ImportRequest) Reset() { *m = ImportRequest{} } @@ -227,7 +229,7 @@ func (*ImportRequest) ProtoMessage() {} func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{11} } type ImportResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,json=err,proto3" json:"Err,omitempty"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` } func (m *ImportResponse) Reset() { *m = ImportResponse{} } @@ -236,10 +238,10 @@ func (*ImportResponse) ProtoMessage() {} func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{12} } type BlockDataRequest struct { - DB string `protobuf:"bytes,1,opt,name=DB,json=dB,proto3" json:"DB,omitempty"` - Frame string `protobuf:"bytes,2,opt,name=Frame,json=frame,proto3" json:"Frame,omitempty"` - Slice uint64 `protobuf:"varint,3,opt,name=Slice,json=slice,proto3" json:"Slice,omitempty"` - Block uint64 `protobuf:"varint,4,opt,name=Block,json=block,proto3" json:"Block,omitempty"` + DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` + Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` + Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` + Block uint64 `protobuf:"varint,4,opt,name=Block,proto3" json:"Block,omitempty"` } func (m *BlockDataRequest) Reset() { *m = BlockDataRequest{} } @@ -248,8 +250,8 @@ func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{13} } type BlockDataResponse struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs,json=profileIDs" json:"ProfileIDs,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` } func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } @@ -258,7 +260,7 @@ func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{14} } type Cache struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs,json=bitmapIDs" json:"BitmapIDs,omitempty"` + BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` } func (m *Cache) Reset() { *m = Cache{} } @@ -267,7 +269,7 @@ func (*Cache) ProtoMessage() {} func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptorInternal, []int{15} } type MaxSlicesResponse struct { - MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices,json=maxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` + MaxSlices map[string]uint64 `protobuf:"bytes,1,rep,name=MaxSlices" json:"MaxSlices,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } func (m *MaxSlicesResponse) Reset() { *m = MaxSlicesResponse{} } @@ -449,6 +451,11 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintInternal(dAtA, i, uint64(m.ProfileID)) } + if m.Timestamp != 0 { + dAtA[i] = 0x18 + i++ + i = encodeVarintInternal(dAtA, i, uint64(m.Timestamp)) + } return i, nil } @@ -814,6 +821,24 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintInternal(dAtA, i, uint64(j8)) i += copy(dAtA[i:], dAtA9[:j8]) } + if len(m.Timestamps) > 0 { + dAtA11 := make([]byte, len(m.Timestamps)*10) + var j10 int + for _, num1 := range m.Timestamps { + num := uint64(num1) + for num >= 1<<7 { + dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j10++ + } + dAtA11[j10] = uint8(num) + j10++ + } + dAtA[i] = 0x32 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j10)) + i += copy(dAtA[i:], dAtA11[:j10]) + } return i, nil } @@ -897,26 +922,9 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.BitmapIDs) > 0 { - dAtA11 := make([]byte, len(m.BitmapIDs)*10) - var j10 int - for _, num := range m.BitmapIDs { - for num >= 1<<7 { - dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80) - num >>= 7 - j10++ - } - dAtA11[j10] = uint8(num) - j10++ - } - dAtA[i] = 0xa - i++ - i = encodeVarintInternal(dAtA, i, uint64(j10)) - i += copy(dAtA[i:], dAtA11[:j10]) - } - if len(m.ProfileIDs) > 0 { - dAtA13 := make([]byte, len(m.ProfileIDs)*10) + dAtA13 := make([]byte, len(m.BitmapIDs)*10) var j12 int - for _, num := range m.ProfileIDs { + for _, num := range m.BitmapIDs { for num >= 1<<7 { dAtA13[j12] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -925,11 +933,28 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { dAtA13[j12] = uint8(num) j12++ } - dAtA[i] = 0x12 + dAtA[i] = 0xa i++ i = encodeVarintInternal(dAtA, i, uint64(j12)) i += copy(dAtA[i:], dAtA13[:j12]) } + if len(m.ProfileIDs) > 0 { + dAtA15 := make([]byte, len(m.ProfileIDs)*10) + var j14 int + for _, num := range m.ProfileIDs { + for num >= 1<<7 { + dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j14++ + } + dAtA15[j14] = uint8(num) + j14++ + } + dAtA[i] = 0x12 + i++ + i = encodeVarintInternal(dAtA, i, uint64(j14)) + i += copy(dAtA[i:], dAtA15[:j14]) + } return i, nil } @@ -949,21 +974,21 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { var l int _ = l if len(m.BitmapIDs) > 0 { - dAtA15 := make([]byte, len(m.BitmapIDs)*10) - var j14 int + dAtA17 := make([]byte, len(m.BitmapIDs)*10) + var j16 int for _, num := range m.BitmapIDs { for num >= 1<<7 { - dAtA15[j14] = uint8(uint64(num)&0x7f | 0x80) + dAtA17[j16] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j14++ + j16++ } - dAtA15[j14] = uint8(num) - j14++ + dAtA17[j16] = uint8(num) + j16++ } dAtA[i] = 0xa i++ - i = encodeVarintInternal(dAtA, i, uint64(j14)) - i += copy(dAtA[i:], dAtA15[:j14]) + i = encodeVarintInternal(dAtA, i, uint64(j16)) + i += copy(dAtA[i:], dAtA17[:j16]) } return i, nil } @@ -1089,6 +1114,9 @@ func (m *Bit) Size() (n int) { if m.ProfileID != 0 { n += 1 + sovInternal(uint64(m.ProfileID)) } + if m.Timestamp != 0 { + n += 1 + sovInternal(uint64(m.Timestamp)) + } return n } @@ -1248,6 +1276,13 @@ func (m *ImportRequest) Size() (n int) { } n += 1 + sovInternal(uint64(l)) + l } + if len(m.Timestamps) > 0 { + l = 0 + for _, e := range m.Timestamps { + l += sovInternal(uint64(e)) + } + n += 1 + sovInternal(uint64(l)) + l + } return n } @@ -1797,6 +1832,25 @@ func (m *Bit) Unmarshal(dAtA []byte) error { break } } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamp", wireType) + } + m.Timestamp = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Timestamp |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipInternal(dAtA[iNdEx:]) @@ -2947,6 +3001,68 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { } else { return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) } + case 6: + if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthInternal + } + postIndex := iNdEx + packedLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + for iNdEx < postIndex { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Timestamps = append(m.Timestamps, v) + } + } else if wireType == 0 { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowInternal + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Timestamps = append(m.Timestamps, v) + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Timestamps", wireType) + } default: iNdEx = preIndex skippy, err := skipInternal(dAtA[iNdEx:]) @@ -3743,51 +3859,50 @@ var ( func init() { proto.RegisterFile("internal.proto", fileDescriptorInternal) } var fileDescriptorInternal = []byte{ - // 721 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xdd, 0x6a, 0x13, 0x41, - 0x14, 0x76, 0xf6, 0x27, 0x9b, 0x3d, 0x69, 0x43, 0x3a, 0x44, 0x59, 0x8a, 0x84, 0xb0, 0xa8, 0x44, - 0xc1, 0x16, 0xea, 0x8d, 0x88, 0x50, 0xdc, 0xa6, 0x62, 0x91, 0x4a, 0x3b, 0xad, 0xde, 0x6f, 0xd2, - 0x69, 0xbb, 0x74, 0xff, 0x3a, 0x33, 0x2b, 0xe6, 0xd2, 0x0b, 0x2f, 0xf4, 0x09, 0x04, 0x5f, 0xc4, - 0x47, 0xf0, 0xd2, 0x47, 0x90, 0xfa, 0x22, 0x32, 0xb3, 0x33, 0xd9, 0x94, 0x62, 0xed, 0x85, 0x97, - 0xdf, 0x77, 0x72, 0xe6, 0x7c, 0xe7, 0x7c, 0xe7, 0x6c, 0xa0, 0x9b, 0xe4, 0x82, 0xb2, 0x3c, 0x4e, - 0xd7, 0x4a, 0x56, 0x88, 0x02, 0xb7, 0x0d, 0x0e, 0x1f, 0x80, 0x35, 0x8e, 0xf0, 0x10, 0x3a, 0x87, - 0x49, 0x46, 0xf7, 0xab, 0x38, 0x17, 0x55, 0x16, 0xa0, 0x21, 0x1a, 0xf9, 0xa4, 0x23, 0x1a, 0x2a, - 0x7c, 0x08, 0xee, 0x4b, 0x16, 0x67, 0xf4, 0x06, 0x3f, 0x8d, 0xa0, 0x15, 0x25, 0x22, 0x8b, 0x4b, - 0x8c, 0xc1, 0x89, 0x12, 0xc1, 0x03, 0x34, 0xb4, 0x47, 0x0e, 0x71, 0x26, 0x89, 0xe0, 0xf8, 0x1e, - 0xb8, 0x2f, 0x84, 0x60, 0x3c, 0xb0, 0x86, 0xf6, 0xa8, 0xb3, 0xd1, 0x5d, 0x9b, 0x4b, 0x93, 0x34, - 0x71, 0x63, 0x19, 0x0c, 0xd7, 0xc0, 0xd9, 0x8b, 0x13, 0x86, 0x7b, 0x60, 0xbf, 0xa6, 0x33, 0x55, - 0xc5, 0x21, 0xf6, 0x19, 0x9d, 0xe1, 0x3e, 0xb8, 0x5b, 0x45, 0x95, 0x8b, 0xc0, 0x52, 0x9c, 0x3b, - 0x95, 0x20, 0xdc, 0x04, 0x3b, 0x4a, 0x04, 0x5e, 0x85, 0x76, 0x5d, 0x7a, 0x67, 0xac, 0x73, 0xda, - 0x13, 0x8d, 0xf1, 0x5d, 0xf0, 0xf7, 0x58, 0x71, 0x9c, 0xa4, 0x74, 0x67, 0xac, 0x93, 0xfd, 0xd2, - 0x10, 0xe1, 0x26, 0x78, 0x3a, 0x8a, 0xbb, 0x60, 0xcd, 0xd3, 0xad, 0x64, 0x7c, 0x43, 0xc5, 0x5f, - 0x10, 0x38, 0x12, 0x2f, 0x4a, 0xf6, 0x6b, 0xc9, 0x18, 0x9c, 0xc3, 0x59, 0x49, 0x75, 0x51, 0x47, - 0xcc, 0x4a, 0x35, 0xc6, 0x03, 0xc1, 0x92, 0xfc, 0xe4, 0x5d, 0x9c, 0x56, 0x34, 0xb0, 0xeb, 0x31, - 0xf2, 0x86, 0x92, 0x7a, 0xdf, 0x26, 0xb9, 0xa8, 0xe3, 0x4e, 0xad, 0xb7, 0x32, 0x84, 0x8c, 0x46, - 0x45, 0x91, 0xd6, 0x51, 0x77, 0x88, 0x46, 0x6d, 0xe2, 0x4f, 0x0c, 0x11, 0xae, 0x83, 0x27, 0xb5, - 0xec, 0xc6, 0x65, 0xa3, 0x1e, 0x5d, 0xa7, 0xfe, 0x3b, 0x82, 0xa5, 0xfd, 0x8a, 0xb2, 0x19, 0xa1, - 0xe7, 0x15, 0xe5, 0x42, 0x0e, 0x61, 0x1c, 0xe9, 0x26, 0xac, 0xa3, 0x48, 0x8e, 0x5d, 0xc5, 0x55, - 0x13, 0x3e, 0x71, 0xcf, 0x25, 0xc0, 0x77, 0xa0, 0x75, 0x90, 0x26, 0x53, 0xca, 0x03, 0x5b, 0x59, - 0xdc, 0xe2, 0x0a, 0x49, 0x1f, 0xf4, 0x34, 0xb9, 0x92, 0xde, 0x26, 0x6d, 0x3d, 0x6a, 0x2e, 0x95, - 0xcb, 0x05, 0xe2, 0x22, 0xce, 0x4a, 0xa5, 0xdc, 0x26, 0xbe, 0x30, 0x04, 0x0e, 0xc0, 0x33, 0xab, - 0xd5, 0x52, 0x95, 0xbc, 0xf3, 0x1a, 0xca, 0x5a, 0x84, 0x66, 0x85, 0xa0, 0x81, 0xa7, 0x5e, 0x6c, - 0x31, 0x85, 0xc2, 0x8f, 0x08, 0x96, 0xb5, 0x74, 0x5e, 0x16, 0x39, 0xa7, 0xd2, 0x81, 0x6d, 0xc6, - 0x8c, 0x03, 0x94, 0x31, 0xbc, 0x0e, 0x1e, 0xa1, 0xbc, 0x4a, 0x85, 0x31, 0xf1, 0x76, 0x33, 0x06, - 0x93, 0x5b, 0xa5, 0x82, 0x78, 0xac, 0xfe, 0x15, 0x7e, 0xbc, 0xd0, 0x80, 0xad, 0x32, 0x56, 0x9a, - 0x0c, 0x1d, 0x69, 0x7a, 0x0a, 0x3f, 0x21, 0xe8, 0x2c, 0xbc, 0x83, 0x47, 0xe6, 0x04, 0x94, 0x88, - 0xce, 0x46, 0xaf, 0x49, 0xae, 0x79, 0xd2, 0xaa, 0xf7, 0x12, 0x2f, 0x01, 0x7a, 0xa3, 0x17, 0x03, - 0xe5, 0xd2, 0x2c, 0xb9, 0xf6, 0xa6, 0xe6, 0x82, 0x59, 0x92, 0x26, 0x6e, 0x29, 0x83, 0x72, 0x46, - 0x5b, 0xa7, 0x71, 0x7e, 0x42, 0x8f, 0xf4, 0x70, 0xbd, 0x69, 0x0d, 0xc3, 0xcf, 0x08, 0x96, 0x77, - 0xb2, 0xb2, 0x60, 0xe2, 0x1a, 0x1f, 0xd5, 0x1d, 0x1b, 0x1f, 0x8f, 0xd5, 0x51, 0xf7, 0xc1, 0x55, - 0x3e, 0xaa, 0x3d, 0x74, 0x88, 0xab, 0x6c, 0x54, 0x3b, 0xa6, 0xaf, 0x47, 0xda, 0x28, 0x0d, 0xf6, - 0xcd, 0x39, 0x71, 0x3c, 0x00, 0x98, 0xdf, 0x13, 0x0f, 0x5c, 0x15, 0x86, 0xf9, 0x41, 0xf1, 0x30, - 0x84, 0xae, 0x91, 0xf2, 0x37, 0x5f, 0xc2, 0x23, 0xe8, 0x45, 0x69, 0x31, 0x3d, 0x1b, 0xc7, 0x22, - 0xfe, 0x1f, 0x8a, 0xfb, 0xe0, 0xaa, 0xf7, 0xf4, 0xbd, 0xb8, 0x13, 0x09, 0xc2, 0x7d, 0x58, 0x59, - 0xa8, 0xa2, 0xc5, 0x5c, 0x6a, 0x0e, 0x5d, 0xdf, 0x9c, 0x75, 0xa5, 0xb9, 0xfb, 0xe0, 0x6e, 0xc5, - 0xd3, 0xd3, 0x7f, 0x3c, 0x13, 0x7e, 0x43, 0xb0, 0xb2, 0x1b, 0x7f, 0xa8, 0x6f, 0x64, 0x5e, 0xfa, - 0x15, 0xf8, 0x73, 0x52, 0x9f, 0xe5, 0xa3, 0xc6, 0xe9, 0x2b, 0xbf, 0x6f, 0x98, 0xed, 0x5c, 0xb0, - 0x19, 0xf1, 0x33, 0x83, 0x57, 0x9f, 0x43, 0xf7, 0x72, 0x50, 0xce, 0xf8, 0xec, 0xf2, 0xd7, 0xa7, - 0x0f, 0xee, 0x7b, 0xf5, 0x95, 0xd0, 0x1f, 0x4c, 0x05, 0x9e, 0x59, 0x4f, 0x51, 0xd4, 0xfb, 0x71, - 0x31, 0x40, 0x3f, 0x2f, 0x06, 0xe8, 0xd7, 0xc5, 0x00, 0x7d, 0xfd, 0x3d, 0xb8, 0x35, 0x69, 0xa9, - 0xbf, 0x87, 0x27, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0x8b, 0x4d, 0xcc, 0x03, 0x30, 0x06, 0x00, - 0x00, + // 706 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6a, 0x14, 0x4f, + 0x10, 0xff, 0xf7, 0xce, 0xcc, 0x7e, 0xd4, 0x26, 0xcb, 0xa6, 0xd9, 0xbf, 0x0c, 0x41, 0x96, 0xa1, + 0x51, 0x19, 0x05, 0x13, 0x88, 0x17, 0x11, 0x41, 0x9c, 0x6c, 0xc4, 0x45, 0x12, 0x92, 0x4e, 0xf4, + 0xe6, 0x61, 0x4c, 0xda, 0x64, 0xc8, 0x7c, 0xd9, 0xd3, 0x23, 0xee, 0xd1, 0x83, 0x17, 0x9f, 0x40, + 0xf0, 0x09, 0x7c, 0x03, 0x1f, 0xc1, 0xa3, 0x8f, 0x20, 0xf1, 0x45, 0xa4, 0x3f, 0xe6, 0x23, 0x11, + 0x63, 0x0e, 0xde, 0xa6, 0x7e, 0xd5, 0xd5, 0x55, 0xbf, 0xfa, 0x55, 0xf5, 0xc0, 0x28, 0x4a, 0x05, + 0xe3, 0x69, 0x18, 0xaf, 0xe5, 0x3c, 0x13, 0x19, 0xee, 0x57, 0x36, 0xb9, 0x05, 0x9d, 0x59, 0x80, + 0x3d, 0x18, 0x1e, 0x44, 0x09, 0xdb, 0x2b, 0xc3, 0x54, 0x94, 0x89, 0x8b, 0x3c, 0xe4, 0x0f, 0x68, + 0x1b, 0x22, 0xb7, 0xc1, 0x79, 0xc2, 0xc3, 0x84, 0x5d, 0xe1, 0x68, 0x00, 0xdd, 0x20, 0x12, 0x49, + 0x98, 0x63, 0x0c, 0x76, 0x10, 0x89, 0xc2, 0x45, 0x9e, 0xe5, 0xdb, 0x54, 0x7d, 0xe3, 0x1b, 0xe0, + 0x3c, 0x16, 0x82, 0x17, 0x6e, 0xc7, 0xb3, 0xfc, 0xe1, 0xc6, 0x68, 0xad, 0x2e, 0x4d, 0xc2, 0x54, + 0x3b, 0xc9, 0x1a, 0xd8, 0xbb, 0x61, 0xc4, 0xf1, 0x18, 0xac, 0x67, 0x6c, 0xa1, 0xb2, 0xd8, 0x54, + 0x7e, 0xe2, 0x09, 0x38, 0x9b, 0x59, 0x99, 0x0a, 0xb7, 0xa3, 0x30, 0x6d, 0x90, 0x97, 0x60, 0x05, + 0x91, 0xc0, 0xab, 0xd0, 0xd7, 0xa9, 0xe7, 0x33, 0x13, 0x53, 0xdb, 0xf8, 0x3a, 0x0c, 0x76, 0x79, + 0xf6, 0x3a, 0x8a, 0xd9, 0x7c, 0x66, 0x82, 0x1b, 0x40, 0x7a, 0x25, 0x87, 0x42, 0x84, 0x49, 0xee, + 0x5a, 0x1e, 0xf2, 0x2d, 0xda, 0x00, 0xe4, 0x11, 0xf4, 0xcc, 0x51, 0x3c, 0x82, 0x4e, 0x7d, 0x79, + 0x67, 0x3e, 0xbb, 0x22, 0x9f, 0x8f, 0x08, 0x6c, 0xf9, 0xd5, 0x26, 0x34, 0xd0, 0x84, 0x30, 0xd8, + 0x07, 0x8b, 0x9c, 0x99, 0x92, 0xd4, 0xb7, 0x6c, 0xf2, 0xbe, 0xe0, 0x51, 0x7a, 0xfc, 0x22, 0x8c, + 0x4b, 0xa6, 0xea, 0x19, 0xd0, 0x36, 0x24, 0xeb, 0x7d, 0x1e, 0xa5, 0x42, 0xfb, 0x6d, 0xcd, 0xa6, + 0x06, 0xa4, 0x37, 0xc8, 0xb2, 0x58, 0x7b, 0x1d, 0x0f, 0xf9, 0x7d, 0xda, 0x00, 0x64, 0x1d, 0x7a, + 0xb2, 0x96, 0xed, 0x30, 0x6f, 0xaa, 0x47, 0x97, 0x55, 0xff, 0x15, 0xc1, 0xd2, 0x5e, 0xc9, 0xf8, + 0x82, 0xb2, 0x37, 0x25, 0x2b, 0x84, 0x6c, 0xc2, 0x2c, 0x30, 0x24, 0xe4, 0xfc, 0x4c, 0xc0, 0x51, + 0x7e, 0x45, 0x62, 0x40, 0xb5, 0x81, 0xaf, 0x41, 0x77, 0x3f, 0x8e, 0x0e, 0x59, 0xe1, 0x5a, 0x6a, + 0x00, 0x8c, 0x25, 0x55, 0x32, 0xdd, 0x2c, 0x54, 0xe9, 0x7d, 0x5a, 0xdb, 0xe7, 0x75, 0x70, 0x2e, + 0xe8, 0x80, 0x5d, 0xe8, 0x55, 0x83, 0xd7, 0x55, 0x99, 0x2a, 0x53, 0xe6, 0xa2, 0x2c, 0xc9, 0x04, + 0x73, 0x7b, 0xea, 0x46, 0x63, 0x91, 0xf7, 0x08, 0x96, 0x4d, 0xe9, 0x45, 0x9e, 0xa5, 0x05, 0x93, + 0x0a, 0x6c, 0x71, 0x5e, 0x29, 0xb0, 0xc5, 0x39, 0x5e, 0x87, 0x1e, 0x65, 0x45, 0x19, 0x8b, 0x4a, + 0xc4, 0xff, 0x9b, 0x36, 0x54, 0xb1, 0x65, 0x2c, 0x68, 0x75, 0x0a, 0xdf, 0x6d, 0x11, 0xb0, 0x54, + 0xc4, 0x4a, 0x13, 0x61, 0x3c, 0x0d, 0x27, 0xf2, 0x01, 0xc1, 0xb0, 0x75, 0x0f, 0xf6, 0xab, 0x05, + 0x51, 0x45, 0x0c, 0x37, 0xc6, 0x4d, 0xb0, 0xc6, 0x69, 0xb5, 0x40, 0x4b, 0x80, 0x76, 0xcc, 0x60, + 0xa0, 0x1d, 0x29, 0x96, 0x5c, 0x8a, 0x2a, 0x67, 0x4b, 0x2c, 0x09, 0x53, 0xed, 0x94, 0x3d, 0xda, + 0x3c, 0x09, 0xd3, 0x63, 0x76, 0x64, 0x9a, 0x5b, 0x99, 0xe4, 0x0b, 0x82, 0xe5, 0x79, 0x92, 0x67, + 0x5c, 0x5c, 0xa2, 0xa3, 0xda, 0xf2, 0x4a, 0x47, 0xbd, 0xf2, 0x13, 0x70, 0x94, 0x72, 0x6a, 0x0e, + 0x6d, 0xaa, 0x0d, 0x35, 0x63, 0x66, 0xb7, 0xa4, 0x8c, 0x52, 0xe0, 0x06, 0xc0, 0x53, 0x80, 0x7a, + 0xb9, 0x0a, 0xd7, 0x51, 0xee, 0x16, 0x22, 0xfd, 0xb5, 0xac, 0x85, 0xdb, 0xf5, 0x2c, 0xdf, 0xa2, + 0x2d, 0x84, 0x10, 0x18, 0x55, 0xa5, 0xfe, 0x49, 0x37, 0x72, 0x04, 0xe3, 0x20, 0xce, 0x0e, 0x4f, + 0x67, 0xa1, 0x08, 0xff, 0x05, 0xa3, 0x09, 0x38, 0xea, 0x3e, 0xb3, 0x4f, 0xda, 0x20, 0x7b, 0xb0, + 0xd2, 0xca, 0x62, 0x8a, 0x39, 0x47, 0x1e, 0x5d, 0x4e, 0xbe, 0x73, 0x91, 0x3c, 0xb9, 0x09, 0xce, + 0x66, 0x78, 0x78, 0xf2, 0x97, 0x6b, 0xc8, 0x67, 0x04, 0x2b, 0xdb, 0xe1, 0x3b, 0xbd, 0x35, 0x75, + 0xea, 0xa7, 0x30, 0xa8, 0x41, 0xb3, 0xb6, 0x77, 0x9a, 0x49, 0xf8, 0xed, 0x7c, 0x83, 0x6c, 0xa5, + 0x82, 0x2f, 0x68, 0x13, 0xbc, 0xfa, 0x10, 0x46, 0xe7, 0x9d, 0xb2, 0xc7, 0xa7, 0xcd, 0xeb, 0x74, + 0xaa, 0x9f, 0xdb, 0xb7, 0xea, 0x15, 0x31, 0xcf, 0xad, 0x32, 0x1e, 0x74, 0xee, 0xa3, 0x60, 0xfc, + 0xed, 0x6c, 0x8a, 0xbe, 0x9f, 0x4d, 0xd1, 0x8f, 0xb3, 0x29, 0xfa, 0xf4, 0x73, 0xfa, 0xdf, 0xab, + 0xae, 0xfa, 0xb9, 0xdc, 0xfb, 0x15, 0x00, 0x00, 0xff, 0xff, 0xe9, 0x7d, 0x09, 0xe4, 0x6e, 0x06, + 0x00, 0x00, } diff --git a/internal/internal.proto b/internal/internal.proto index a26cd1f02..bab66e6bf 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -23,6 +23,7 @@ message Pair { message Bit { uint64 BitmapID = 1; uint64 ProfileID = 2; + int64 Timestamp = 3; } message Profile { @@ -71,6 +72,7 @@ message ImportRequest { uint64 Slice = 3; repeated uint64 BitmapIDs = 4; repeated uint64 ProfileIDs = 5; + repeated int64 Timestamps = 6; } message ImportResponse { diff --git a/pilosa.go b/pilosa.go index d6c07bb0c..a93dc241c 100644 --- a/pilosa.go +++ b/pilosa.go @@ -75,3 +75,6 @@ func decodeProfile(pb *internal.Profile) *Profile { return p } + +// TimeFormat is the go-style time format used to parse string dates. +const TimeFormat = "2006-01-02T15:04"