scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

merged with master

Browse source
This commit is contained in:
Yuce Tekol 2017-08-25 23:59:40 +03:00
commit 3367c69959
No known key found for this signature in database
GPG key ID: CB59E46D2FB90573
12 changed files with 759 additions and 142 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

142
executor.go
View file

@ -161,6 +161,9 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
indexTag := fmt.Sprintf("index:%s", index)
// Special handling for mutation and top-n calls.
switch c.Name {
case "Average":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeAverage(ctx, index, c, slices, opt)
case "ClearBit":
return e.executeClearBit(ctx, index, c, opt)
case "Count":
@ -174,6 +177,9 @@ func (e *Executor) executeCall(ctx context.Context, index string, c *pql.Call, s
return nil, e.executeSetRowAttrs(ctx, index, c, opt)
case "SetColumnAttrs":
return nil, e.executeSetColumnAttrs(ctx, index, c, opt)
case "Sum":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeSum(ctx, index, c, slices, opt)
case "TopN":
e.Holder.Stats.CountWithCustomTags(c.Name, 1, 1.0, []string{indexTag})
return e.executeTopN(ctx, index, c, slices, opt)
@ -202,6 +208,41 @@ func (e *Executor) validateCallArgs(c *pql.Call) error {
return nil
}
// executeAverage executes an average() call.
func (e *Executor) executeAverage(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (int64, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return 0, errors.New("Average(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return 0, errors.New("Average(): field required")
}
if len(c.Children) > 1 {
return 0, errors.New("Average() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeSumCountSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(SumCount)
return other.Add(v.(SumCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
other, _ := result.(SumCount)
if other.Count == 0 {
return 0, nil
}
return other.Sum / other.Count, nil
}
// executeBitmapCall executes a call that returns a bitmap.
func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (*Bitmap, error) {
// Execute calls in bulk on each remote node and merge.
@ -287,6 +328,77 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, index string, c *
}
}
// executeSum executes a sum() call.
func (e *Executor) executeSum(ctx context.Context, index string, c *pql.Call, slices []uint64, opt *ExecOptions) (int64, error) {
if frame, _ := c.Args["frame"]; frame == "" {
return 0, errors.New("Sum(): frame required")
} else if field, _ := c.Args["field"]; field == "" {
return 0, errors.New("Sum(): field required")
}
if len(c.Children) > 1 {
return 0, errors.New("Sum() only accepts a single bitmap input")
}
// Execute calls in bulk on each remote node and merge.
mapFn := func(slice uint64) (interface{}, error) {
return e.executeSumCountSlice(ctx, index, c, slice)
}
// Merge returned results at coordinating node.
reduceFn := func(prev, v interface{}) interface{} {
other, _ := prev.(SumCount)
return other.Add(v.(SumCount))
}
result, err := e.mapReduce(ctx, index, slices, c, opt, mapFn, reduceFn)
if err != nil {
return 0, err
}
other, _ := result.(SumCount)
return other.Sum, nil
}
// executeSumCountSlice executes calculates the sum & count for fields on a slice.
func (e *Executor) executeSumCountSlice(ctx context.Context, index string, c *pql.Call, slice uint64) (SumCount, error) {
var filter *Bitmap
if len(c.Children) == 1 {
bm, err := e.executeBitmapCallSlice(ctx, index, c.Children[0], slice)
if err != nil {
return SumCount{}, err
}
filter = bm
}
frameName, _ := c.Args["frame"].(string)
fieldName, _ := c.Args["field"].(string)
frame := e.Holder.Frame(index, frameName)
if frame == nil {
return SumCount{}, nil
}
field := frame.Field(fieldName)
if field == nil {
return SumCount{}, nil
}
view := e.Holder.Fragment(index, frameName, ViewFieldPrefix+fieldName, slice)
if view == nil {
return SumCount{}, nil
}
vsum, vcount, err := view.FieldSum(filter, field.BitDepth())
if err != nil {
return SumCount{}, err
}
return SumCount{
Sum: int64(vsum) + (int64(vcount) * field.Min),
Count: int64(vcount),
}, nil
}
// executeTopN executes a TopN() call.
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
@ -1186,6 +1298,8 @@ func (e *Executor) exec(ctx context.Context, node *Node, index string, q *pql.Qu
var err error
switch call.Name {
case "Average", "Sum":
v, err = decodeSumCount(pb.Results[i].GetSumCount()), nil
case "TopN":
v, err = decodePairs(pb.Results[i].GetPairs()), nil
case "Count":
@ -1304,7 +1418,6 @@ func (e *Executor) mapper(ctx context.Context, ch chan mapResponse, nodes []*Nod
if n.Host == e.Host {
resp.result, resp.err = e.mapperLocal(ctx, nodeSlices, mapFn, reduceFn)
} else if !opt.Remote {
results, err := e.exec(ctx, n, index, &pql.Query{Calls: []*pql.Call{c}}, nodeSlices, opt)
if len(results) > 0 {
resp.result = results[0]
@ -1423,3 +1536,30 @@ func needsSlices(calls []*pql.Call) bool {
}
return false
}
// SumCount represents a grouping of sum & count for Sum() and Average() calls.
type SumCount struct {
Sum int64 `json:"sum"`
Count int64 `json:"count"`
}
func (sc *SumCount) Add(other SumCount) SumCount {
return SumCount{
Sum: sc.Sum + other.Sum,
Count: sc.Count + other.Count,
}
}
func encodeSumCount(sc SumCount) *internal.SumCount {
return &internal.SumCount{
Sum: sc.Sum,
Count: sc.Count,
}
}
func decodeSumCount(pb *internal.SumCount) SumCount {
return SumCount{
Sum: pb.Sum,
Count: pb.Count,
}
}

122
executor_test.go
View file

@ -441,6 +441,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
}
})
}
func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -578,7 +579,128 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
}}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
}
// Ensure a Sum() query can be executed.
func TestExecutor_Execute_Sum(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=f, foo=20, bar=2000, columnID=0)
SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, columnID=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("NoFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if result[0] != int64(200) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Sum(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if result[0] != int64(80) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
}
// Ensure a Average() query can be executed.
func TestExecutor_Execute_Average(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 10, Max: 100},
{Name: "bar", Type: pilosa.FieldTypeInt, Min: 0, Max: 100000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := idx.CreateFrame("other", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: 0, Max: 1000},
},
}); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=20, bar=2000, columnID=0)
SetFieldValue(frame=f, foo=30, columnID=`+strconv.Itoa(SliceWidth)+`)
SetFieldValue(frame=f, foo=40, columnID=`+strconv.Itoa(SliceWidth+2)+`)
SetFieldValue(frame=f, foo=50, columnID=`+strconv.Itoa((5*SliceWidth)+100)+`)
SetFieldValue(frame=f, foo=60, columnID=`+strconv.Itoa(SliceWidth+1)+`)
SetFieldValue(frame=other, foo=1000, columnID=0)
`), nil, nil); err != nil {
t.Fatal(err)
}
t.Run("NoFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Average(frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if result[0] != int64(40) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("WithFilter", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Average(Bitmap(frame=f, rowID=0), frame=f, field=foo)`), nil, nil); err != nil {
t.Fatal(err)
} else if result[0] != int64(30) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
}
// Ensure a range query can be executed.

31
fragment.go
View file

@ -536,6 +536,37 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
return changed, nil
}
// FieldSum returns the sum of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) {
f.mu.Lock()
defer f.mu.Unlock()
// Compute count based on the existance bit.
row := f.row(uint64(bitDepth), true, true)
if filter != nil {
row = row.Intersect(filter)
}
count = row.Count()
// Compute the sum based on the bit count of each row multiplied by the
// place value of each row. For example, 10 bits in the 1's place plus
// 4 bits in the 2's place plus 3 bits in the 4's place equals a total
// sum of 30:
//
// 10*(2^0) + 4*(2^1) + 3*(2^2) = 30
//
for i := uint(0); i < bitDepth; i++ {
row := f.row(uint64(i), true, true)
if filter != nil {
row = row.Intersect(filter)
}
sum += (1 << i) * row.Count()
}
return sum, count, nil
}
func (f *Fragment) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
switch op {
case RangeOpEQ:

80
fragment_test.go
View file

@ -216,6 +216,45 @@ func TestFragment_SetFieldValue(t *testing.T) {
})
}
// Ensure a fragment can sum field values.
func TestFragment_FieldSum(t *testing.T) {
const bitDepth = 16
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
t.Fatal(err)
}
t.Run("NoFilter", func(t *testing.T) {
if sum, n, err := f.FieldSum(nil, bitDepth); err != nil {
t.Fatal(err)
} else if n != 4 {
t.Fatalf("unexpected count: %d", n)
} else if sum != 3800 {
t.Fatalf("unexpected sum: %d", sum)
}
})
t.Run("WithFilter", func(t *testing.T) {
if sum, n, err := f.FieldSum(pilosa.NewBitmap(2000, 4000, 5000), bitDepth); err != nil {
t.Fatal(err)
} else if n != 2 {
t.Fatalf("unexpected count: %d", n)
} else if sum != 600 {
t.Fatalf("unexpected sum: %d", sum)
}
})
}
// Ensure a fragment query for matching fields.
func TestFragment_FieldRange(t *testing.T) {
const bitDepth = 16
@ -940,28 +979,27 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
}
func TestFragment_Snapshot_Run(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set bits on the fragment.
for i := uint64(1); i < 3; i++ {
if _, err := f.SetBit(1000, i); err != nil {
t.Fatal(err)
}
}
// Set bits on the fragment.
for i := uint64(1); i < 3; i++ {
if _, err := f.SetBit(1000, i); err != nil {
t.Fatal(err)
}
}
// Snapshot bitmap and verify data.
if err := f.Snapshot(); err != nil {
t.Fatal(err)
} else if n := f.Row(1000).Count(); n != 2 {
t.Fatalf("unexpected count: %d", n)
}
// Snapshot bitmap and verify data.
if err := f.Snapshot(); err != nil {
t.Fatal(err)
} else if n := f.Row(1000).Count(); n != 2 {
t.Fatalf("unexpected count: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if n := f.Row(1000).Count(); n != 2 {
t.Fatalf("unexpected count (reopen): %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if n := f.Row(1000).Count(); n != 2 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}

32
frame.go
View file

@ -124,11 +124,15 @@ func (f *Frame) MaxSlice() uint64 {
f.mu.Lock()
defer f.mu.Unlock()
view := f.views[ViewStandard]
if view == nil {
return 0
var max uint64
for _, view := range f.views {
if view.name == ViewInverse {
continue
} else if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max {
max = viewMaxSlice
}
}
return view.MaxSlice()
return max
}
// MaxInverseSlice returns the max inverse slice in the frame.
@ -639,6 +643,26 @@ func (f *Frame) SetFieldValue(columnID uint64, name string, value int64) (change
return view.SetFieldValue(columnID, field.BitDepth(), baseValue)
}
// FieldSum returns the sum and count for a field.
// An optional filtering bitmap can be provided.
func (f *Frame) FieldSum(filter *Bitmap, name string) (sum, count int64, err error) {
field := f.Field(name)
if field == nil {
return 0, 0, ErrFieldNotFound
}
view := f.View(ViewFieldPrefix + name)
if view == nil {
return 0, 0, nil
}
vsum, vcount, err := view.FieldSum(filter, field.BitDepth())
if err != nil {
return 0, 0, err
}
return int64(vsum) + (int64(vcount) * field.Min), int64(vcount), nil
}
func (f *Frame) FieldRange(name, op string, predicate int64) (*Bitmap, error) {
// Retrieve and validate field.
field := f.Field(name)

12
gossip/gossip.go
View file

@ -69,6 +69,12 @@ func (g *GossipNodeSet) Open() error {
return err
}
g.memberlist = ml
g.broadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return ml.NumMembers()
},
RetransmitMult: 3,
}
// attach to gossip seed node
nodes := []*pilosa.Node{&pilosa.Node{Host: g.config.gossipSeed}} //TODO: support a list of seeds
@ -76,12 +82,6 @@ func (g *GossipNodeSet) Open() error {
if err != nil {
return err
}
g.broadcasts = &memberlist.TransmitLimitedQueue{
NumNodes: func() int {
return ml.NumMembers()
},
RetransmitMult: 3,
}
return nil
}

2
handler.go
View file

@ -1467,6 +1467,8 @@ func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse {
pb.Results[i].Bitmap = encodeBitmap(result)
case []Pair:
pb.Results[i].Pairs = encodePairs(result)
case SumCount:
pb.Results[i].SumCount = encodeSumCount(result)
case uint64:
pb.Results[i].N = result
case bool:

345
internal/public.pb.go
View file

@ -11,6 +11,7 @@
It has these top-level messages:
Bitmap
Pair
SumCount
Bit
ColumnAttrSet
Attr
@ -66,6 +67,16 @@ func (m *Pair) String() string { return proto.CompactTextString(m) }
func (*Pair) ProtoMessage() {}
func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} }
type SumCount struct {
Sum int64 `protobuf:"varint,1,opt,name=Sum,proto3" json:"Sum,omitempty"`
Count int64 `protobuf:"varint,2,opt,name=Count,proto3" json:"Count,omitempty"`
}
func (m *SumCount) Reset() { *m = SumCount{} }
func (m *SumCount) String() string { return proto.CompactTextString(m) }
func (*SumCount) ProtoMessage() {}
func (*SumCount) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
type Bit struct {
RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"`
ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"`
@ -75,7 +86,7 @@ type Bit struct {
func (m *Bit) Reset() { *m = Bit{} }
func (m *Bit) String() string { return proto.CompactTextString(m) }
func (*Bit) ProtoMessage() {}
func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} }
type ColumnAttrSet struct {
ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
@ -85,7 +96,7 @@ type ColumnAttrSet struct {
func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} }
func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) }
func (*ColumnAttrSet) ProtoMessage() {}
func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} }
func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} }
func (m *ColumnAttrSet) GetAttrs() []*Attr {
if m != nil {
@ -106,7 +117,7 @@ type Attr struct {
func (m *Attr) Reset() { *m = Attr{} }
func (m *Attr) String() string { return proto.CompactTextString(m) }
func (*Attr) ProtoMessage() {}
func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{4} }
func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} }
type AttrMap struct {
Attrs []*Attr `protobuf:"bytes,1,rep,name=Attrs" json:"Attrs,omitempty"`
@ -115,7 +126,7 @@ type AttrMap struct {
func (m *AttrMap) Reset() { *m = AttrMap{} }
func (m *AttrMap) String() string { return proto.CompactTextString(m) }
func (*AttrMap) ProtoMessage() {}
func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{5} }
func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} }
func (m *AttrMap) GetAttrs() []*Attr {
if m != nil {
@ -136,7 +147,7 @@ type QueryRequest struct {
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
func (m *QueryRequest) String() string { return proto.CompactTextString(m) }
func (*QueryRequest) ProtoMessage() {}
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} }
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} }
type QueryResponse struct {
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
@ -147,7 +158,7 @@ type QueryResponse struct {
func (m *QueryResponse) Reset() { *m = QueryResponse{} }
func (m *QueryResponse) String() string { return proto.CompactTextString(m) }
func (*QueryResponse) ProtoMessage() {}
func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{7} }
func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} }
func (m *QueryResponse) GetResults() []*QueryResult {
if m != nil {
@ -164,16 +175,17 @@ func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet {
}
type QueryResult struct {
Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"`
N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"`
Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"`
Bitmap *Bitmap `protobuf:"bytes,1,opt,name=Bitmap" json:"Bitmap,omitempty"`
N uint64 `protobuf:"varint,2,opt,name=N,proto3" json:"N,omitempty"`
Pairs []*Pair `protobuf:"bytes,3,rep,name=Pairs" json:"Pairs,omitempty"`
SumCount *SumCount `protobuf:"bytes,5,opt,name=SumCount" json:"SumCount,omitempty"`
Changed bool `protobuf:"varint,4,opt,name=Changed,proto3" json:"Changed,omitempty"`
}
func (m *QueryResult) Reset() { *m = QueryResult{} }
func (m *QueryResult) String() string { return proto.CompactTextString(m) }
func (*QueryResult) ProtoMessage() {}
func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{8} }
func (*QueryResult) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} }
func (m *QueryResult) GetBitmap() *Bitmap {
if m != nil {
@ -189,6 +201,13 @@ func (m *QueryResult) GetPairs() []*Pair {
return nil
}
func (m *QueryResult) GetSumCount() *SumCount {
if m != nil {
return m.SumCount
}
return nil
}
type ImportRequest struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
@ -201,11 +220,12 @@ type ImportRequest struct {
func (m *ImportRequest) Reset() { *m = ImportRequest{} }
func (m *ImportRequest) String() string { return proto.CompactTextString(m) }
func (*ImportRequest) ProtoMessage() {}
func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{9} }
func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{10} }
func init() {
proto.RegisterType((*Bitmap)(nil), "internal.Bitmap")
proto.RegisterType((*Pair)(nil), "internal.Pair")
proto.RegisterType((*SumCount)(nil), "internal.SumCount")
proto.RegisterType((*Bit)(nil), "internal.Bit")
proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet")
proto.RegisterType((*Attr)(nil), "internal.Attr")
@ -290,6 +310,34 @@ func (m *Pair) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *SumCount) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
if err != nil {
return nil, err
}
return dAtA[:n], nil
}
func (m *SumCount) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
_ = l
if m.Sum != 0 {
dAtA[i] = 0x8
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Sum))
}
if m.Count != 0 {
dAtA[i] = 0x10
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Count))
}
return i, nil
}
func (m *Bit) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
@ -624,6 +672,16 @@ func (m *QueryResult) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.SumCount != nil {
dAtA[i] = 0x2a
i++
i = encodeVarintPublic(dAtA, i, uint64(m.SumCount.Size()))
n6, err := m.SumCount.MarshalTo(dAtA[i:])
if err != nil {
return 0, err
}
i += n6
}
return i, nil
}
@ -660,56 +718,56 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
}
if len(m.RowIDs) > 0 {
dAtA7 := make([]byte, len(m.RowIDs)*10)
var j6 int
dAtA8 := make([]byte, len(m.RowIDs)*10)
var j7 int
for _, num := range m.RowIDs {
for num >= 1<<7 {
dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80)
dAtA8[j7] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j6++
j7++
}
dAtA7[j6] = uint8(num)
j6++
dAtA8[j7] = uint8(num)
j7++
}
dAtA[i] = 0x22
i++
i = encodeVarintPublic(dAtA, i, uint64(j6))
i += copy(dAtA[i:], dAtA7[:j6])
i = encodeVarintPublic(dAtA, i, uint64(j7))
i += copy(dAtA[i:], dAtA8[:j7])
}
if len(m.ColumnIDs) > 0 {
dAtA9 := make([]byte, len(m.ColumnIDs)*10)
var j8 int
dAtA10 := make([]byte, len(m.ColumnIDs)*10)
var j9 int
for _, num := range m.ColumnIDs {
for num >= 1<<7 {
dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80)
dAtA10[j9] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j8++
j9++
}
dAtA9[j8] = uint8(num)
j8++
dAtA10[j9] = uint8(num)
j9++
}
dAtA[i] = 0x2a
i++
i = encodeVarintPublic(dAtA, i, uint64(j8))
i += copy(dAtA[i:], dAtA9[:j8])
i = encodeVarintPublic(dAtA, i, uint64(j9))
i += copy(dAtA[i:], dAtA10[:j9])
}
if len(m.Timestamps) > 0 {
dAtA11 := make([]byte, len(m.Timestamps)*10)
var j10 int
dAtA12 := make([]byte, len(m.Timestamps)*10)
var j11 int
for _, num1 := range m.Timestamps {
num := uint64(num1)
for num >= 1<<7 {
dAtA11[j10] = uint8(uint64(num)&0x7f | 0x80)
dAtA12[j11] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
j10++
j11++
}
dAtA11[j10] = uint8(num)
j10++
dAtA12[j11] = uint8(num)
j11++
}
dAtA[i] = 0x32
i++
i = encodeVarintPublic(dAtA, i, uint64(j10))
i += copy(dAtA[i:], dAtA11[:j10])
i = encodeVarintPublic(dAtA, i, uint64(j11))
i += copy(dAtA[i:], dAtA12[:j11])
}
return i, nil
}
@ -772,6 +830,18 @@ func (m *Pair) Size() (n int) {
return n
}
func (m *SumCount) Size() (n int) {
var l int
_ = l
if m.Sum != 0 {
n += 1 + sovPublic(uint64(m.Sum))
}
if m.Count != 0 {
n += 1 + sovPublic(uint64(m.Count))
}
return n
}
func (m *Bit) Size() (n int) {
var l int
_ = l
@ -910,6 +980,10 @@ func (m *QueryResult) Size() (n int) {
if m.Changed {
n += 2
}
if m.SumCount != nil {
l = m.SumCount.Size()
n += 1 + l + sovPublic(uint64(l))
}
return n
}
@ -1195,6 +1269,94 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *SumCount) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
preIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: SumCount: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: SumCount: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Sum", wireType)
}
m.Sum = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Sum |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Count", wireType)
}
m.Count = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
m.Count |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
if iNdEx > l {
return io.ErrUnexpectedEOF
}
return nil
}
func (m *Bit) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
@ -2161,6 +2323,39 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
}
}
m.Changed = bool(v != 0)
case 5:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field SumCount", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPublic
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.SumCount == nil {
m.SumCount = &SumCount{}
}
if err := m.SumCount.Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -2603,42 +2798,44 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 589 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x5d, 0x8e, 0xd3, 0x30,
0x10, 0xc6, 0x4d, 0xfa, 0x37, 0xed, 0xae, 0x56, 0x16, 0x3f, 0x11, 0x42, 0x55, 0x14, 0xf1, 0x90,
0xa7, 0xae, 0xb4, 0x1c, 0x00, 0x91, 0xb6, 0x2b, 0x45, 0x88, 0x15, 0xb8, 0x0b, 0xef, 0xd9, 0xd6,
0x5a, 0x22, 0x25, 0x71, 0x70, 0x1c, 0xb1, 0x3d, 0x00, 0x27, 0xe0, 0x85, 0x1b, 0xc0, 0x21, 0x38,
0x00, 0x8f, 0x1c, 0x01, 0x95, 0x8b, 0xa0, 0xb1, 0xe3, 0x26, 0xe5, 0x01, 0xf1, 0xe6, 0xef, 0x1b,
0xcf, 0x64, 0x3e, 0x7f, 0x33, 0x81, 0x69, 0x59, 0xdf, 0x64, 0xe9, 0x66, 0x5e, 0x4a, 0xa1, 0x04,
0x1d, 0xa5, 0x85, 0xe2, 0xb2, 0x48, 0xb2, 0x20, 0x82, 0x41, 0x94, 0xaa, 0x3c, 0x29, 0x29, 0x05,
0x37, 0x4a, 0x55, 0xe5, 0x11, 0xdf, 0x09, 0x5d, 0xa6, 0xcf, 0xf4, 0x29, 0xf4, 0x5f, 0x28, 0x25,
0x2b, 0xaf, 0xe7, 0x3b, 0xe1, 0xe4, 0xe2, 0x74, 0x6e, 0xf3, 0xe6, 0x48, 0x33, 0x13, 0x0c, 0xe6,
0xe0, 0xbe, 0x4e, 0x52, 0x49, 0xcf, 0xc0, 0x79, 0xc9, 0x77, 0x1e, 0xf1, 0x49, 0xe8, 0x32, 0x3c,
0xd2, 0xfb, 0xd0, 0x5f, 0x88, 0xba, 0x50, 0x5e, 0x4f, 0x73, 0x06, 0x04, 0x6f, 0xc1, 0x89, 0x52,
0x85, 0x41, 0x26, 0x3e, 0xc6, 0xcb, 0x26, 0xc1, 0x00, 0xfa, 0x18, 0x46, 0x0b, 0x91, 0xd5, 0x79,
0x11, 0x2f, 0x9b, 0xac, 0x03, 0xa6, 0x4f, 0x60, 0x7c, 0x9d, 0xe6, 0xbc, 0x52, 0x49, 0x5e, 0x7a,
0x8e, 0x4f, 0x42, 0x87, 0xb5, 0x44, 0xb0, 0x82, 0x13, 0x73, 0x13, 0xbb, 0x5a, 0x73, 0x45, 0x4f,
0xa1, 0x77, 0xa8, 0xde, 0x8b, 0x97, 0xff, 0xa9, 0xe6, 0x1b, 0x01, 0x17, 0x4f, 0x5d, 0x39, 0x63,
0x23, 0x87, 0x82, 0x7b, 0xbd, 0x2b, 0x79, 0xd3, 0x97, 0x3e, 0x53, 0x1f, 0x26, 0x6b, 0x25, 0xd3,
0xe2, 0xf6, 0x5d, 0x92, 0xd5, 0x5c, 0x77, 0x35, 0x66, 0x5d, 0x0a, 0x15, 0xc5, 0x85, 0x32, 0x61,
0x57, 0x37, 0x7d, 0xc0, 0xa8, 0x28, 0x12, 0x22, 0x33, 0xc1, 0xbe, 0x4f, 0xc2, 0x11, 0x6b, 0x09,
0x3a, 0x03, 0xb8, 0xcc, 0x44, 0xd2, 0xe4, 0x0e, 0x7c, 0x12, 0x12, 0xd6, 0x61, 0x82, 0x73, 0x18,
0x62, 0xa7, 0xaf, 0x92, 0xb2, 0xd5, 0x46, 0xfe, 0xa5, 0xed, 0x3b, 0x81, 0xe9, 0x9b, 0x9a, 0xcb,
0x1d, 0xe3, 0x1f, 0x6a, 0x5e, 0x69, 0x0f, 0x34, 0x6e, 0x54, 0x1a, 0x40, 0x1f, 0xc2, 0x60, 0x9d,
0xa5, 0x1b, 0x6e, 0x5e, 0xca, 0x65, 0x0d, 0x42, 0xad, 0xed, 0x0b, 0x57, 0x5a, 0xeb, 0x88, 0x75,
0x29, 0xcc, 0x64, 0x3c, 0x17, 0xca, 0x8a, 0x69, 0x10, 0x0d, 0x60, 0xba, 0xba, 0xdb, 0x64, 0xf5,
0x96, 0x9b, 0xd4, 0x81, 0x8e, 0x1e, 0x71, 0x58, 0xbd, 0xc1, 0x7a, 0x0e, 0x87, 0xa6, 0x7a, 0x87,
0x0a, 0x3e, 0x13, 0x38, 0x69, 0xda, 0xaf, 0x4a, 0x51, 0x54, 0x1c, 0x3d, 0x5a, 0x49, 0x69, 0x3d,
0x5a, 0x49, 0x49, 0xcf, 0x61, 0xc8, 0x78, 0x55, 0x67, 0xca, 0xda, 0xfc, 0xa0, 0x7d, 0x0a, 0x9b,
0x5b, 0x67, 0x8a, 0xd9, 0x5b, 0xf4, 0x39, 0x9c, 0x1e, 0x8d, 0x0d, 0xea, 0xc2, 0xbc, 0x47, 0x6d,
0xde, 0x51, 0x9c, 0xfd, 0x75, 0x3d, 0xf8, 0x44, 0x60, 0xd2, 0xa9, 0x4c, 0x43, 0xbb, 0x52, 0xba,
0xad, 0xc9, 0xc5, 0x59, 0x5b, 0xc8, 0xf0, 0xcc, 0xae, 0xdc, 0x14, 0xc8, 0x55, 0x33, 0x4c, 0xe4,
0x0a, 0x2d, 0xc4, 0x35, 0xb2, 0xdf, 0xef, 0x58, 0x88, 0x34, 0x33, 0x41, 0xea, 0xc1, 0x70, 0xf1,
0x3e, 0x29, 0x6e, 0xf9, 0x56, 0x0f, 0xd3, 0x88, 0x59, 0x18, 0x7c, 0x25, 0x70, 0x12, 0xe7, 0xa5,
0x90, 0xaa, 0xe3, 0x6e, 0x5c, 0x6c, 0xf9, 0x9d, 0x75, 0x57, 0x03, 0x64, 0x2f, 0x65, 0x92, 0x9b,
0x31, 0x1e, 0x33, 0x03, 0x90, 0xd5, 0x2e, 0x6b, 0x57, 0x5d, 0x66, 0x80, 0xf6, 0x13, 0xd7, 0xb2,
0xf2, 0x5c, 0x33, 0x09, 0x06, 0xe1, 0xdc, 0xda, 0xad, 0xac, 0xbc, 0xbe, 0x0e, 0xb5, 0x04, 0xce,
0xed, 0x61, 0x2d, 0xd1, 0x6b, 0x27, 0x74, 0x58, 0x87, 0x89, 0xce, 0x7e, 0xec, 0x67, 0xe4, 0xe7,
0x7e, 0x46, 0x7e, 0xed, 0x67, 0xe4, 0xcb, 0xef, 0xd9, 0xbd, 0x9b, 0x81, 0xfe, 0x2f, 0x3d, 0xfb,
0x13, 0x00, 0x00, 0xff, 0xff, 0xd9, 0x81, 0x5d, 0x51, 0xa7, 0x04, 0x00, 0x00,
// 621 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x8e, 0xd3, 0x40,
0x10, 0xa5, 0x63, 0x27, 0x71, 0x2a, 0x99, 0x51, 0xd4, 0xe2, 0x63, 0x21, 0x14, 0x59, 0x16, 0x0b,
0xaf, 0x32, 0x52, 0x38, 0x00, 0xc2, 0x49, 0x46, 0xb2, 0x10, 0x23, 0xe8, 0x0c, 0xec, 0x3d, 0x49,
0x6b, 0xb0, 0xe4, 0x1f, 0xed, 0xb6, 0x98, 0x9c, 0x83, 0x0d, 0x37, 0x80, 0x0b, 0xb0, 0xe3, 0x00,
0x2c, 0x39, 0x02, 0x0a, 0x17, 0x41, 0xd5, 0xed, 0x8e, 0x1d, 0x16, 0x88, 0x5d, 0xbd, 0x57, 0x5d,
0xe5, 0xfa, 0xbc, 0x32, 0x4c, 0xca, 0xfa, 0x26, 0x4d, 0xb6, 0xf3, 0x52, 0x14, 0xb2, 0xa0, 0x4e,
0x92, 0x4b, 0x2e, 0xf2, 0x38, 0xf5, 0x43, 0x18, 0x84, 0x89, 0xcc, 0xe2, 0x92, 0x52, 0xb0, 0xc3,
0x44, 0x56, 0x2e, 0xf1, 0xac, 0xc0, 0x66, 0xca, 0xa6, 0x4f, 0xa1, 0xff, 0x42, 0x4a, 0x51, 0xb9,
0x3d, 0xcf, 0x0a, 0xc6, 0x8b, 0xf3, 0xb9, 0x89, 0x9b, 0x23, 0xcd, 0xb4, 0xd3, 0x9f, 0x83, 0xfd,
0x3a, 0x4e, 0x04, 0x9d, 0x82, 0xf5, 0x92, 0xef, 0x5d, 0xe2, 0x91, 0xc0, 0x66, 0x68, 0xd2, 0xfb,
0xd0, 0x5f, 0x16, 0x75, 0x2e, 0xdd, 0x9e, 0xe2, 0x34, 0xf0, 0x17, 0xe0, 0x6c, 0xea, 0x4c, 0xd9,
0x18, 0xb3, 0xa9, 0x33, 0x15, 0x63, 0x31, 0x34, 0x4f, 0x63, 0x2c, 0x13, 0xf3, 0x16, 0xac, 0x30,
0x91, 0xe8, 0x64, 0xc5, 0xc7, 0x68, 0xd5, 0x7c, 0x44, 0x03, 0xfa, 0x18, 0x9c, 0x65, 0x91, 0xd6,
0x59, 0x1e, 0xad, 0x9a, 0x2f, 0x1d, 0x31, 0x7d, 0x02, 0xa3, 0xeb, 0x24, 0xe3, 0x95, 0x8c, 0xb3,
0xd2, 0xb5, 0x54, 0xca, 0x96, 0xf0, 0xd7, 0x70, 0xa6, 0x5f, 0x62, 0x27, 0x1b, 0x2e, 0xe9, 0x39,
0xf4, 0x8e, 0xd9, 0x7b, 0xd1, 0xea, 0x3f, 0x27, 0xf0, 0x95, 0x80, 0x8d, 0x56, 0x77, 0x04, 0x23,
0x3d, 0x02, 0x0a, 0xf6, 0xf5, 0xbe, 0xe4, 0x4d, 0x5d, 0xca, 0xa6, 0x1e, 0x8c, 0x37, 0x52, 0x24,
0xf9, 0xed, 0xbb, 0x38, 0xad, 0xb9, 0xaa, 0x6a, 0xc4, 0xba, 0x14, 0x76, 0x14, 0xe5, 0x52, 0xbb,
0x6d, 0x55, 0xf4, 0x11, 0x63, 0x47, 0x61, 0x51, 0xa4, 0xda, 0xd9, 0xf7, 0x48, 0xe0, 0xb0, 0x96,
0xa0, 0x33, 0x80, 0xcb, 0xb4, 0x88, 0x9b, 0xd8, 0x81, 0x47, 0x02, 0xc2, 0x3a, 0x8c, 0x7f, 0x01,
0x43, 0xac, 0xf4, 0x55, 0x5c, 0xb6, 0xbd, 0x91, 0x7f, 0xf5, 0xf6, 0x9d, 0xc0, 0xe4, 0x4d, 0xcd,
0xc5, 0x9e, 0xf1, 0x0f, 0x35, 0xaf, 0xd4, 0x0e, 0x14, 0x6e, 0xba, 0xd4, 0x80, 0x3e, 0x84, 0xc1,
0x26, 0x4d, 0xb6, 0x5c, 0x4f, 0xca, 0x66, 0x0d, 0xc2, 0x5e, 0xdb, 0x09, 0x57, 0xaa, 0x57, 0x87,
0x75, 0x29, 0x8c, 0x64, 0x3c, 0x2b, 0xa4, 0x69, 0xa6, 0x41, 0xd4, 0x87, 0xc9, 0xfa, 0x6e, 0x9b,
0xd6, 0x3b, 0xae, 0x43, 0x07, 0xca, 0x7b, 0xc2, 0x61, 0xf6, 0x06, 0x2b, 0xed, 0x0e, 0x75, 0xf6,
0x0e, 0xe5, 0x7f, 0x22, 0x70, 0xd6, 0x94, 0x5f, 0x95, 0x45, 0x5e, 0x71, 0xdc, 0xd1, 0x5a, 0x08,
0xb3, 0xa3, 0xb5, 0x10, 0xf4, 0x02, 0x86, 0x8c, 0x57, 0x75, 0x2a, 0xcd, 0x9a, 0x1f, 0xb4, 0xa3,
0x30, 0xb1, 0x75, 0x2a, 0x99, 0x79, 0x45, 0x9f, 0xc3, 0xf9, 0x89, 0x6c, 0xb0, 0x2f, 0x8c, 0x7b,
0xd4, 0xc6, 0x9d, 0xf8, 0xd9, 0x5f, 0xcf, 0xfd, 0x6f, 0x04, 0xc6, 0x9d, 0xcc, 0x34, 0x30, 0x67,
0xa8, 0xca, 0x1a, 0x2f, 0xa6, 0x6d, 0x22, 0xcd, 0x33, 0x73, 0xa6, 0x13, 0x20, 0x57, 0x8d, 0x98,
0xc8, 0x15, 0xae, 0x10, 0x4f, 0xcf, 0x7c, 0xbf, 0xb3, 0x42, 0xa4, 0x99, 0x76, 0x52, 0x17, 0x86,
0xcb, 0xf7, 0x71, 0x7e, 0xcb, 0x77, 0x4a, 0x4c, 0x0e, 0x33, 0x90, 0xce, 0xdb, 0x53, 0x54, 0xd3,
0x1f, 0x2f, 0x68, 0x9b, 0xc2, 0x78, 0xd8, 0xf1, 0x8d, 0xff, 0x85, 0xc0, 0x59, 0x94, 0x95, 0x85,
0x90, 0x1d, 0x35, 0x44, 0xf9, 0x8e, 0xdf, 0x19, 0x35, 0x28, 0x80, 0xec, 0xa5, 0x88, 0x33, 0x2d,
0xfb, 0x11, 0xd3, 0x00, 0x59, 0xa5, 0x0a, 0xa5, 0x02, 0x9b, 0x69, 0xa0, 0xf6, 0x8f, 0x67, 0x5c,
0xb9, 0xb6, 0x56, 0x8e, 0x46, 0xa8, 0x73, 0x73, 0xc5, 0x95, 0xdb, 0x57, 0xae, 0x96, 0x40, 0x9d,
0x1f, 0xcf, 0x18, 0xb5, 0x61, 0x05, 0x16, 0xeb, 0x30, 0xe1, 0xf4, 0xc7, 0x61, 0x46, 0x7e, 0x1e,
0x66, 0xe4, 0xd7, 0x61, 0x46, 0x3e, 0xff, 0x9e, 0xdd, 0xbb, 0x19, 0xa8, 0x7f, 0xdf, 0xb3, 0x3f,
0x01, 0x00, 0x00, 0xff, 0xff, 0x27, 0x5d, 0xef, 0xb2, 0x0b, 0x05, 0x00, 0x00,
}

6
internal/public.proto
View file

@ -12,6 +12,11 @@ message Pair {
uint64 Count = 2;
}
message SumCount {
int64 Sum = 1;
int64 Count = 2;
}
message Bit {
uint64 RowID = 1;
uint64 ColumnID = 2;
@ -55,6 +60,7 @@ message QueryResult {
Bitmap Bitmap = 1;
uint64 N = 2;
repeated Pair Pairs = 3;
SumCount SumCount = 5;
bool Changed = 4;
}

55
roaring/roaring.go
View file

@ -2508,7 +2508,7 @@ func differenceRunBitmap(a, b *container) *container {
func differenceRunIterator(a *container, itr containerIterator) *container {
output := &container{runs: make([]interval16, 0, a.n)}
output := &container{runs: make([]interval16, 0, a.n), container_type: ContainerRun}
vb, eof := itr.next()
j := 0
@ -2576,7 +2576,7 @@ func differenceRunRun(a, b *container) *container {
alen := len(a.runs)
blen := len(b.runs)
output := &container{runs: make([]interval16, 0, alen+blen)} // TODO allocate max then truncate? or something else
output := &container{runs: make([]interval16, 0, alen+blen), container_type: ContainerRun} // TODO allocate max then truncate? or something else
// cardinality upper bound: sum of number of runs
// each B-run could split an A-run in two, up to len(b.runs) times
@ -3156,8 +3156,9 @@ func xorArrayRun(a, b *container) *container {
} else if va > vb.start {
if va < vb.last {
output.n += output.runAppendInterval(interval16{start: vb.start, last: va - 1})
vb.start = va + 1
i++
vb.start = va + 1
if vb.start > vb.last {
j++
}
@ -3166,15 +3167,22 @@ func xorArrayRun(a, b *container) *container {
j++
} else { // va == vb.last
vb.last--
if vb.start < vb.last {
if vb.start <= vb.last {
output.n += output.runAppendInterval(vb)
}
j++
i++
}
} else {
vb.start++
} else { // we know va == vb.start
if vb.start == maxContainerVal { // protect overflow
j++
} else {
vb.start++
if vb.start > vb.last {
j++
}
}
i++
}
}
@ -3222,9 +3230,15 @@ func xorCompare(x *xorstm) (r1 interval16, has_data bool) {
r1 = interval16{start: x.va.start, last: x.vb.start - 1}
has_data = true
}
x.va.start = x.vb.last + 1
if x.va.start > x.va.last {
if x.vb.last == maxContainerVal { // Check for overflow
x.va_valid = false
} else {
x.va.start = x.vb.last + 1
if x.va.start > x.va.last {
x.va_valid = false
}
}
} else if x.vb.start <= x.va.start && x.vb.last >= x.va.last { //va inside
@ -3234,26 +3248,39 @@ func xorCompare(x *xorstm) (r1 interval16, has_data bool) {
has_data = true
}
x.vb.start = x.va.last + 1
if x.vb.start > x.vb.last {
if x.va.last == maxContainerVal { //check for overflow
x.vb_valid = false
} else {
x.vb.start = x.va.last + 1
if x.vb.start > x.vb.last {
x.vb_valid = false
}
}
} else if x.va.start < x.vb.start && x.va.last <= x.vb.last { //va first overlap
x.va_valid = false
r1 = interval16{start: x.va.start, last: x.vb.start - 1}
has_data = true
x.vb.start = x.va.last + 1
if x.vb.start > x.vb.last {
if x.va.last == maxContainerVal { // check for overflow
x.vb_valid = false
} else {
x.vb.start = x.va.last + 1
if x.vb.start > x.vb.last {
x.vb_valid = false
}
}
} else if x.vb.start < x.va.start && x.vb.last <= x.va.last { //vb first overlap
x.vb_valid = false
r1 = interval16{start: x.vb.start, last: x.va.start - 1}
has_data = true
x.va.start = x.vb.last + 1
if x.va.start > x.va.last {
if x.vb.last == maxContainerVal { // check for overflow
x.va_valid = false
} else {
x.va.start = x.vb.last + 1
if x.va.start > x.va.last {
x.va_valid = false
}
}
}
return

61
roaring/roaring_internal_test.go
View file

@ -1711,31 +1711,43 @@ func TestWriteReadRun(t *testing.T) {
}
func TestXorArrayRun(t *testing.T) {
a := &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray}
b := &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun}
exp := []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}
//ret := xorArrayRun(a, b)
ret := xor(a, b)
if !reflect.DeepEqual(ret.array, exp) {
t.Fatalf("test #1 expected %v, but got %v", exp, ret.array)
tests := []struct {
a *container
b *container
exp *container
}{
{
a: &container{array: []uint16{1, 5, 10, 11, 12}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun},
exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 13, 15, 16}, container_type: ContainerArray, n: 12},
}, {
a: &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 2, last: 10}, {start: 12, last: 13}, {start: 15, last: 16}}, container_type: ContainerRun},
exp: &container{array: []uint16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}, container_type: ContainerArray, n: 12},
}, {
a: &container{array: []uint16{65535}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 65534, last: 65535}}, container_type: ContainerRun},
exp: &container{array: []uint16{65534}, container_type: ContainerArray, n: 1},
}, {
a: &container{array: []uint16{65535}, container_type: ContainerArray},
b: &container{runs: []interval16{{start: 65535, last: 65535}}, container_type: ContainerRun},
exp: &container{array: []uint16{}, container_type: ContainerArray, n: 0},
},
}
ret = xor(b, a)
if !reflect.DeepEqual(ret.array, exp) {
t.Fatalf("test #2 expected %v, but got %v", exp, ret.array)
}
c := &container{array: []uint16{1, 5, 10, 11, 12, 13, 14}, container_type: ContainerArray}
// exp = []int16{1, 2, 3, 4, 6, 7, 8, 9, 11, 14, 15, 16}
expr := []interval16{{start: 1, last: 4}, {start: 6, last: 9}, {start: 11, last: 11}, {start: 14, last: 16}}
ret = xor(b, c)
if !reflect.DeepEqual(ret.runs, expr) {
t.Fatalf("test #3 expected %v, but got %v", exp, ret.runs)
}
ret = xor(c, b)
if !reflect.DeepEqual(ret.runs, expr) {
t.Fatalf("test #4 expected %v, but got %v", exp, ret.array)
for i, test := range tests {
test.a.n = test.a.count()
test.b.n = test.b.count()
ret := xor(test.a, test.b)
if !reflect.DeepEqual(ret, test.exp) {
t.Fatalf("test #%v expected %v, but got %v", i, test.exp, ret)
}
ret = xor(test.b, test.a)
if !reflect.DeepEqual(ret, test.exp) {
t.Fatalf("test #%v.1 expected %v, but got %v", i, test.exp, ret)
}
}
}
//special case that didn't fit the xorrunrun table testing below.
@ -1832,6 +1844,11 @@ func TestXorRunRun(t *testing.T) {
bruns: []interval16{{start: 2, last: 8}, {start: 16, last: 27}, {start: 33, last: 34}},
exp: []interval16{{start: 1, last: 1}, {start: 4, last: 4}, {start: 6, last: 6}, {start: 9, last: 9}, {start: 12, last: 15}, {start: 23, last: 27}, {start: 33, last: 34}},
},
{
aruns: []interval16{{start: 65530, last: 65535}},
bruns: []interval16{{start: 65532, last: 65535}},
exp: []interval16{{start: 65530, last: 65531}},
},
}
for i, test := range tests {
a.runs = test.aruns

13
view.go
View file

@ -300,6 +300,19 @@ func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (chan
return frag.SetFieldValue(columnID, bitDepth, value)
}
// FieldSum returns the sum & count of a field.
func (v *View) FieldSum(filter *Bitmap, bitDepth uint) (sum, count uint64, err error) {
for _, f := range v.Fragments() {
fsum, fcount, err := f.FieldSum(filter, bitDepth)
if err != nil {
return sum, count, err
}
sum += fsum
count += fcount
}
return sum, count, nil
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (v *View) FieldRange(op string, bitDepth uint, predicate uint64) (*Bitmap, error) {
bm := NewBitmap()