From 7c8a27338a332d41e5327c7aed3769f692dc7c3c Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Tue, 16 Feb 2016 17:38:56 -0600 Subject: [PATCH 1/5] added support for returning SetBit effect --- cmd/pilosactl/main.go | 2 +- executor.go | 22 ++++++++++++++-------- fragment.go | 36 +++++++++++++++++++++++------------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/cmd/pilosactl/main.go b/cmd/pilosactl/main.go index 13fd226bd..38f1d0717 100644 --- a/cmd/pilosactl/main.go +++ b/cmd/pilosactl/main.go @@ -303,7 +303,7 @@ func (cmd *ImportCommand) parsePath(path string) ([]pilosa.Bit, error) { // Ignore blank rows. if record[0] == "" { continue - } else if len(record) != 2 { + } else if len(record) < 2 { return nil, fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record)) } diff --git a/executor.go b/executor.go index e94ed677a..7bc2a2991 100644 --- a/executor.go +++ b/executor.go @@ -50,7 +50,7 @@ func (e *Executor) Execute(db string, q *pql.Query, slices []uint64) (interface{ // Ignore slices for set calls. switch root := q.Root.(type) { case *pql.SetBit: - return nil, e.executeSetBit(db, root) + return e.executeSetBit(db, root) case *pql.SetBitmapAttrs: return nil, e.executeSetBitmapAttrs(db, root) case *pql.SetProfileAttrs: @@ -315,27 +315,33 @@ func (e *Executor) executeProfile(db string, c *pql.Profile) (*Profile, error) { } // executeSetBit executes a SetBit() call. -func (e *Executor) executeSetBit(db string, c *pql.SetBit) error { +func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool,error){ slice := c.ProfileID / SliceWidth - + ret :=false for _, node := range e.Cluster.SliceNodes(slice) { // Update locally if host matches. if node.Host == e.Host { f, err := e.Index().CreateFragmentIfNotExists(db, c.Frame, slice) if err != nil { - return fmt.Errorf("fragment: %s", err) + return false,fmt.Errorf("fragment: %s", err) + } + err,val :=f.SetBit(c.ID, c.ProfileID) + if err != nil{ + return false, err + } + if val{ + ret = true } - f.SetBit(c.ID, c.ProfileID) continue } // Forward call to remote node otherwise. if _, err := e.exec(node, db, &pql.Query{Root: c}, nil); err != nil { - // FIXME: Handle errors more gracefully. - return err + return false,err } + fmt.Println("NEED TO IMPLEMENT REMOTE SETBIT") } - return nil + return ret,nil } // executeSetBitmapAttrs executes a SetBitmapAttrs() call. diff --git a/fragment.go b/fragment.go index ed6ec90a3..c47aae119 100644 --- a/fragment.go +++ b/fragment.go @@ -13,6 +13,7 @@ import ( "syscall" "time" "unsafe" + "reflect" "github.com/gogo/protobuf/proto" "github.com/umbel/pilosa/internal" @@ -312,51 +313,50 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { // SetBit sets a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(bitmapID, profileID uint64) error { +func (f *Fragment) SetBit(bitmapID, profileID uint64) (error,bool) { f.mu.Lock() defer f.mu.Unlock() return f.setBit(bitmapID, profileID) } -func (f *Fragment) setBit(bitmapID, profileID uint64) error { +func (f *Fragment) setBit(bitmapID, profileID uint64) (error,bool) { // Determine the position of the bit in the storage. pos, err := f.pos(bitmapID, profileID) if err != nil { - return err + return err,false } // Write to storage. if err := f.storage.Add(pos); err != nil { - return err + return err,false } - // Update the cache. - f.bitmap(bitmapID).setBit(profileID) - return nil + // Update the cache. + return nil,f.bitmap(bitmapID).setBit(profileID) + } // ClearBit clears a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) ClearBit(bitmapID, profileID uint64) error { +func (f *Fragment) ClearBit(bitmapID, profileID uint64) (error,bool) { f.mu.Lock() defer f.mu.Unlock() // Determine the position of the bit in the storage. pos, err := f.pos(bitmapID, profileID) if err != nil { - return err + return err,false } // Write to storage. if err := f.storage.Remove(pos); err != nil { - return err + return err,false } // Update the cache. - f.bitmap(bitmapID).clearBit(profileID) + return nil, f.bitmap(bitmapID).clearBit(profileID) - return nil } // pos translates the bitmap ID and profile ID into a position in the storage bitmap. @@ -385,7 +385,15 @@ func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interfac if len(fieldValues) > 0 { filters = make(map[interface{}]struct{}) for _, v := range fieldValues { - filters[v] = struct{}{} + switch v.(type){ + case uint64: + i:=int64(v.(uint64)) + filters[i] = struct{}{} + default: + filters[v] = struct{}{} + } + fmt.Println(reflect.TypeOf(v)) + } } @@ -409,6 +417,8 @@ func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interfac } else if attrValue := attr[field]; attrValue == nil { continue } else if _, ok := filters[attrValue]; !ok { + fmt.Println(reflect.TypeOf(attrValue),"==") + continue } } From 6d3d10da2bf4dc917189610657d316dd3fdf5c75 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 17 Feb 2016 11:06:57 -0600 Subject: [PATCH 2/5] changed protobuf to uint; doesn't work --- attr.go | 14 +++-- executor.go | 16 ++--- fragment.go | 35 +++++------ internal/internal.pb.go | 127 ++++++++++++++++++++++++++-------------- internal/internal.proto | 2 +- 5 files changed, 117 insertions(+), 77 deletions(-) diff --git a/attr.go b/attr.go index 2105bf368..a9eab184c 100644 --- a/attr.go +++ b/attr.go @@ -2,6 +2,7 @@ package pilosa import ( "encoding/binary" + "fmt" "sort" "sync" "time" @@ -175,9 +176,14 @@ func encodeAttr(key string, value interface{}) *internal.Attr { case string: pb.StringValue = proto.String(value) case float64: - pb.IntValue = proto.Int64(int64(value)) + fmt.Println("A") + pb.UintValue = proto.Uint64(uint64(value)) + case uint64: + fmt.Println("b") + pb.UintValue = proto.Uint64(value) case int64: - pb.IntValue = proto.Int64(value) + fmt.Println("c") + pb.UintValue = proto.Uint64(uint64(value)) case bool: pb.BoolValue = proto.Bool(value) } @@ -188,8 +194,8 @@ func encodeAttr(key string, value interface{}) *internal.Attr { func decodeAttr(attr *internal.Attr) (key string, value interface{}) { if attr.StringValue != nil { return attr.GetKey(), attr.GetStringValue() - } else if attr.IntValue != nil { - return attr.GetKey(), attr.GetIntValue() + } else if attr.UintValue != nil { + return attr.GetKey(), attr.GetUintValue() } else if attr.BoolValue != nil { return attr.GetKey(), attr.GetBoolValue() } diff --git a/executor.go b/executor.go index 7bc2a2991..3e419c949 100644 --- a/executor.go +++ b/executor.go @@ -315,21 +315,21 @@ func (e *Executor) executeProfile(db string, c *pql.Profile) (*Profile, error) { } // executeSetBit executes a SetBit() call. -func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool,error){ +func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool, error) { slice := c.ProfileID / SliceWidth - ret :=false + ret := false for _, node := range e.Cluster.SliceNodes(slice) { // Update locally if host matches. if node.Host == e.Host { f, err := e.Index().CreateFragmentIfNotExists(db, c.Frame, slice) if err != nil { - return false,fmt.Errorf("fragment: %s", err) + return false, fmt.Errorf("fragment: %s", err) } - err,val :=f.SetBit(c.ID, c.ProfileID) - if err != nil{ + val, err := f.SetBit(c.ID, c.ProfileID) + if err != nil { return false, err } - if val{ + if val { ret = true } continue @@ -337,11 +337,11 @@ func (e *Executor) executeSetBit(db string, c *pql.SetBit) (bool,error){ // Forward call to remote node otherwise. if _, err := e.exec(node, db, &pql.Query{Root: c}, nil); err != nil { - return false,err + return false, err } fmt.Println("NEED TO IMPLEMENT REMOTE SETBIT") } - return ret,nil + return ret, nil } // executeSetBitmapAttrs executes a SetBitmapAttrs() call. diff --git a/fragment.go b/fragment.go index c47aae119..6ce30dbaa 100644 --- a/fragment.go +++ b/fragment.go @@ -7,13 +7,13 @@ import ( "io/ioutil" "log" "os" + "reflect" "sort" "strings" "sync" "syscall" "time" "unsafe" - "reflect" "github.com/gogo/protobuf/proto" "github.com/umbel/pilosa/internal" @@ -313,49 +313,48 @@ func (f *Fragment) bitmap(bitmapID uint64) *Bitmap { // SetBit sets a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(bitmapID, profileID uint64) (error,bool) { +func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() return f.setBit(bitmapID, profileID) } -func (f *Fragment) setBit(bitmapID, profileID uint64) (error,bool) { +func (f *Fragment) setBit(bitmapID, profileID uint64) (bool, error) { // Determine the position of the bit in the storage. pos, err := f.pos(bitmapID, profileID) if err != nil { - return err,false + return false, err } // Write to storage. if err := f.storage.Add(pos); err != nil { - return err,false + return false, err } - // Update the cache. - return nil,f.bitmap(bitmapID).setBit(profileID) + return f.bitmap(bitmapID).setBit(profileID), nil } // ClearBit clears a bit for a given profile & bitmap within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) ClearBit(bitmapID, profileID uint64) (error,bool) { +func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) { f.mu.Lock() defer f.mu.Unlock() // Determine the position of the bit in the storage. pos, err := f.pos(bitmapID, profileID) if err != nil { - return err,false + return false, err } // Write to storage. if err := f.storage.Remove(pos); err != nil { - return err,false + return false, err } // Update the cache. - return nil, f.bitmap(bitmapID).clearBit(profileID) + return f.bitmap(bitmapID).clearBit(profileID), nil } @@ -385,14 +384,8 @@ func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interfac if len(fieldValues) > 0 { filters = make(map[interface{}]struct{}) for _, v := range fieldValues { - switch v.(type){ - case uint64: - i:=int64(v.(uint64)) - filters[i] = struct{}{} - default: - filters[v] = struct{}{} - } - fmt.Println(reflect.TypeOf(v)) + filters[v] = struct{}{} + fmt.Println("B:", reflect.TypeOf(v)) } } @@ -410,6 +403,7 @@ func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interfac // Apply filter, if set. if filters != nil { attr, err := f.BitmapAttrStore.Attrs(bitmapID) + fmt.Println("C:", reflect.TypeOf(attr), attr) if err != nil { return nil, err } else if attr == nil { @@ -417,8 +411,7 @@ func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interfac } else if attrValue := attr[field]; attrValue == nil { continue } else if _, ok := filters[attrValue]; !ok { - fmt.Println(reflect.TypeOf(attrValue),"==") - + fmt.Println("A:", reflect.TypeOf(attrValue)) continue } } diff --git a/internal/internal.pb.go b/internal/internal.pb.go index cb128d7c6..ef49d0d53 100644 --- a/internal/internal.pb.go +++ b/internal/internal.pb.go @@ -1,12 +1,12 @@ -// Code generated by protoc-gen-gogo. -// source: internal/internal.proto +// Code generated by protoc-gen-go. +// source: internal.proto // DO NOT EDIT! /* Package internal is a generated protocol buffer package. It is generated from these files: - internal/internal.proto + internal.proto It has these top-level messages: Bitmap @@ -39,9 +39,10 @@ type Bitmap struct { XXX_unrecognized []byte `json:"-"` } -func (m *Bitmap) Reset() { *m = Bitmap{} } -func (m *Bitmap) String() string { return proto.CompactTextString(m) } -func (*Bitmap) ProtoMessage() {} +func (m *Bitmap) Reset() { *m = Bitmap{} } +func (m *Bitmap) String() string { return proto.CompactTextString(m) } +func (*Bitmap) ProtoMessage() {} +func (*Bitmap) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } func (m *Bitmap) GetChunks() []*Chunk { if m != nil { @@ -63,9 +64,10 @@ type Chunk struct { XXX_unrecognized []byte `json:"-"` } -func (m *Chunk) Reset() { *m = Chunk{} } -func (m *Chunk) String() string { return proto.CompactTextString(m) } -func (*Chunk) ProtoMessage() {} +func (m *Chunk) Reset() { *m = Chunk{} } +func (m *Chunk) String() string { return proto.CompactTextString(m) } +func (*Chunk) ProtoMessage() {} +func (*Chunk) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } func (m *Chunk) GetKey() uint64 { if m != nil && m.Key != nil { @@ -87,9 +89,10 @@ type Pair struct { XXX_unrecognized []byte `json:"-"` } -func (m *Pair) Reset() { *m = Pair{} } -func (m *Pair) String() string { return proto.CompactTextString(m) } -func (*Pair) ProtoMessage() {} +func (m *Pair) Reset() { *m = Pair{} } +func (m *Pair) String() string { return proto.CompactTextString(m) } +func (*Pair) ProtoMessage() {} +func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } func (m *Pair) GetKey() uint64 { if m != nil && m.Key != nil { @@ -111,9 +114,10 @@ type Bit struct { XXX_unrecognized []byte `json:"-"` } -func (m *Bit) Reset() { *m = Bit{} } -func (m *Bit) String() string { return proto.CompactTextString(m) } -func (*Bit) ProtoMessage() {} +func (m *Bit) Reset() { *m = Bit{} } +func (m *Bit) String() string { return proto.CompactTextString(m) } +func (*Bit) ProtoMessage() {} +func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } func (m *Bit) GetBitmapID() uint64 { if m != nil && m.BitmapID != nil { @@ -135,9 +139,10 @@ type Profile struct { XXX_unrecognized []byte `json:"-"` } -func (m *Profile) Reset() { *m = Profile{} } -func (m *Profile) String() string { return proto.CompactTextString(m) } -func (*Profile) ProtoMessage() {} +func (m *Profile) Reset() { *m = Profile{} } +func (m *Profile) String() string { return proto.CompactTextString(m) } +func (*Profile) ProtoMessage() {} +func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } func (m *Profile) GetID() uint64 { if m != nil && m.ID != nil { @@ -156,14 +161,15 @@ func (m *Profile) GetAttrs() []*Attr { type Attr struct { Key *string `protobuf:"bytes,1,req,name=Key" json:"Key,omitempty"` StringValue *string `protobuf:"bytes,2,opt,name=StringValue" json:"StringValue,omitempty"` - IntValue *int64 `protobuf:"varint,3,opt,name=IntValue" json:"IntValue,omitempty"` + UintValue *uint64 `protobuf:"varint,3,opt,name=UintValue" json:"UintValue,omitempty"` BoolValue *bool `protobuf:"varint,4,opt,name=BoolValue" json:"BoolValue,omitempty"` XXX_unrecognized []byte `json:"-"` } -func (m *Attr) Reset() { *m = Attr{} } -func (m *Attr) String() string { return proto.CompactTextString(m) } -func (*Attr) ProtoMessage() {} +func (m *Attr) Reset() { *m = Attr{} } +func (m *Attr) String() string { return proto.CompactTextString(m) } +func (*Attr) ProtoMessage() {} +func (*Attr) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } func (m *Attr) GetKey() string { if m != nil && m.Key != nil { @@ -179,9 +185,9 @@ func (m *Attr) GetStringValue() string { return "" } -func (m *Attr) GetIntValue() int64 { - if m != nil && m.IntValue != nil { - return *m.IntValue +func (m *Attr) GetUintValue() uint64 { + if m != nil && m.UintValue != nil { + return *m.UintValue } return 0 } @@ -198,9 +204,10 @@ type AttrMap struct { XXX_unrecognized []byte `json:"-"` } -func (m *AttrMap) Reset() { *m = AttrMap{} } -func (m *AttrMap) String() string { return proto.CompactTextString(m) } -func (*AttrMap) ProtoMessage() {} +func (m *AttrMap) Reset() { *m = AttrMap{} } +func (m *AttrMap) String() string { return proto.CompactTextString(m) } +func (*AttrMap) ProtoMessage() {} +func (*AttrMap) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } func (m *AttrMap) GetAttrs() []*Attr { if m != nil { @@ -217,9 +224,10 @@ type QueryRequest struct { XXX_unrecognized []byte `json:"-"` } -func (m *QueryRequest) Reset() { *m = QueryRequest{} } -func (m *QueryRequest) String() string { return proto.CompactTextString(m) } -func (*QueryRequest) ProtoMessage() {} +func (m *QueryRequest) Reset() { *m = QueryRequest{} } +func (m *QueryRequest) String() string { return proto.CompactTextString(m) } +func (*QueryRequest) ProtoMessage() {} +func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } func (m *QueryRequest) GetDB() string { if m != nil && m.DB != nil { @@ -258,9 +266,10 @@ type QueryResponse struct { XXX_unrecognized []byte `json:"-"` } -func (m *QueryResponse) Reset() { *m = QueryResponse{} } -func (m *QueryResponse) String() string { return proto.CompactTextString(m) } -func (*QueryResponse) ProtoMessage() {} +func (m *QueryResponse) Reset() { *m = QueryResponse{} } +func (m *QueryResponse) String() string { return proto.CompactTextString(m) } +func (*QueryResponse) ProtoMessage() {} +func (*QueryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } func (m *QueryResponse) GetErr() string { if m != nil && m.Err != nil { @@ -306,9 +315,10 @@ type ImportRequest struct { XXX_unrecognized []byte `json:"-"` } -func (m *ImportRequest) Reset() { *m = ImportRequest{} } -func (m *ImportRequest) String() string { return proto.CompactTextString(m) } -func (*ImportRequest) ProtoMessage() {} +func (m *ImportRequest) Reset() { *m = ImportRequest{} } +func (m *ImportRequest) String() string { return proto.CompactTextString(m) } +func (*ImportRequest) ProtoMessage() {} +func (*ImportRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } func (m *ImportRequest) GetDB() string { if m != nil && m.DB != nil { @@ -350,9 +360,10 @@ type ImportResponse struct { XXX_unrecognized []byte `json:"-"` } -func (m *ImportResponse) Reset() { *m = ImportResponse{} } -func (m *ImportResponse) String() string { return proto.CompactTextString(m) } -func (*ImportResponse) ProtoMessage() {} +func (m *ImportResponse) Reset() { *m = ImportResponse{} } +func (m *ImportResponse) String() string { return proto.CompactTextString(m) } +func (*ImportResponse) ProtoMessage() {} +func (*ImportResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } func (m *ImportResponse) GetErr() string { if m != nil && m.Err != nil { @@ -366,9 +377,10 @@ type Cache struct { XXX_unrecognized []byte `json:"-"` } -func (m *Cache) Reset() { *m = Cache{} } -func (m *Cache) String() string { return proto.CompactTextString(m) } -func (*Cache) ProtoMessage() {} +func (m *Cache) Reset() { *m = Cache{} } +func (m *Cache) String() string { return proto.CompactTextString(m) } +func (*Cache) ProtoMessage() {} +func (*Cache) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } func (m *Cache) GetBitmapIDs() []uint64 { if m != nil { @@ -391,3 +403,32 @@ func init() { proto.RegisterType((*ImportResponse)(nil), "internal.ImportResponse") proto.RegisterType((*Cache)(nil), "internal.Cache") } + +var fileDescriptor0 = []byte{ + // 398 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x84, 0x52, 0x4d, 0x8f, 0xda, 0x30, + 0x14, 0x54, 0x88, 0x03, 0xe4, 0xa5, 0xa4, 0xe0, 0x5e, 0x50, 0x25, 0x54, 0x64, 0x2e, 0xa8, 0x07, + 0x0e, 0xa8, 0x7f, 0xa0, 0x40, 0xab, 0x22, 0x54, 0x44, 0x8b, 0xda, 0x73, 0x23, 0xe4, 0x96, 0xa8, + 0x21, 0xce, 0x3a, 0xce, 0x81, 0x1f, 0xb1, 0xff, 0x79, 0x9f, 0x3f, 0x12, 0xd8, 0x5d, 0x56, 0x7b, + 0x8a, 0x3c, 0x1e, 0xbf, 0x99, 0x79, 0x13, 0x88, 0xd3, 0x5c, 0x71, 0x99, 0x27, 0xd9, 0xac, 0x90, + 0x42, 0x09, 0xda, 0xad, 0xcf, 0xec, 0x1b, 0xb4, 0x17, 0xa9, 0x3a, 0x25, 0x05, 0xfd, 0x00, 0xed, + 0xe5, 0xb1, 0xca, 0xff, 0x97, 0x43, 0x6f, 0xec, 0x4f, 0xa3, 0xf9, 0xdb, 0x59, 0xf3, 0xc8, 0xe0, + 0x74, 0x04, 0xc1, 0x67, 0xa5, 0x64, 0x39, 0x6c, 0x99, 0xfb, 0xf8, 0x72, 0xaf, 0x61, 0x36, 0x81, + 0xc0, 0xf2, 0x22, 0xf0, 0x37, 0xfc, 0x8c, 0x53, 0x5a, 0x53, 0x42, 0x7b, 0x10, 0xfc, 0x4e, 0xb2, + 0x8a, 0x9b, 0x47, 0x84, 0x31, 0x20, 0xbb, 0x24, 0x95, 0xcf, 0x38, 0x4b, 0x51, 0xe5, 0x0a, 0x39, + 0x78, 0x64, 0x1f, 0xc1, 0x47, 0x4b, 0xb4, 0x0f, 0x5d, 0xeb, 0x6c, 0xbd, 0x72, 0xbc, 0x01, 0x84, + 0x3b, 0x29, 0xfe, 0xa6, 0x19, 0x47, 0xc8, 0x72, 0x3f, 0x41, 0xc7, 0x41, 0x14, 0xa0, 0xd5, 0x30, + 0x5f, 0xb1, 0xba, 0x05, 0xa2, 0xbf, 0xd7, 0x2e, 0x42, 0xfa, 0x0e, 0xa2, 0xbd, 0x92, 0x69, 0xfe, + 0xaf, 0xf6, 0xeb, 0x21, 0x88, 0x92, 0xbf, 0xf0, 0xad, 0x85, 0x7c, 0x84, 0x8c, 0x8b, 0x85, 0x10, + 0x99, 0x85, 0x08, 0x42, 0x5d, 0x36, 0x85, 0x8e, 0x9e, 0xf7, 0x1d, 0xb7, 0xd8, 0x28, 0x7b, 0x37, + 0x95, 0x37, 0xf0, 0xe6, 0x47, 0xc5, 0xe5, 0xf9, 0x27, 0xbf, 0xab, 0x78, 0xa9, 0xb4, 0xe9, 0xd5, + 0xc2, 0x19, 0xc0, 0x35, 0x98, 0x3b, 0x13, 0x2d, 0xa4, 0x31, 0xb4, 0xf7, 0x59, 0x7a, 0xe0, 0x25, + 0xea, 0xe2, 0xea, 0xf4, 0x3e, 0x5c, 0xd4, 0xd2, 0xc9, 0xde, 0x7b, 0xd0, 0x73, 0xd3, 0xca, 0x42, + 0xe4, 0x25, 0xd7, 0x81, 0xbe, 0x48, 0x89, 0xf3, 0xb4, 0xf7, 0x71, 0x5d, 0xad, 0xc9, 0x12, 0xcd, + 0xfb, 0x17, 0x2f, 0xae, 0xf2, 0x10, 0xbc, 0xad, 0x4b, 0x85, 0xbe, 0x75, 0x31, 0x7a, 0xf4, 0x13, + 0xdf, 0xa6, 0xaf, 0xc9, 0x95, 0x78, 0x60, 0x18, 0x83, 0x2b, 0x86, 0xbd, 0x61, 0x7f, 0xa0, 0xb7, + 0x3e, 0x15, 0x42, 0xaa, 0x17, 0xd2, 0x7d, 0x95, 0xc9, 0x89, 0xbb, 0x74, 0x78, 0x34, 0xe9, 0x50, + 0xde, 0x55, 0x5b, 0x97, 0x6d, 0x2d, 0x10, 0x8a, 0xaf, 0x9b, 0xb6, 0xad, 0x28, 0x61, 0x23, 0x88, + 0x6b, 0x85, 0x1b, 0x89, 0xd9, 0x7b, 0xfc, 0x91, 0x92, 0xc3, 0x91, 0x3f, 0x1e, 0xa7, 0x9b, 0x20, + 0x0f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x4e, 0xbb, 0x74, 0xfe, 0x03, 0x03, 0x00, 0x00, +} diff --git a/internal/internal.proto b/internal/internal.proto index f40e8d61b..31f8688aa 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -28,7 +28,7 @@ message Profile { message Attr { required string Key = 1; optional string StringValue = 2; - optional int64 IntValue = 3; + optional uint64 UintValue = 3; optional bool BoolValue = 4; } From e1eca36a9d907bf5c4dc09c8755fd13c57ca3780 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 17 Feb 2016 10:59:17 -0700 Subject: [PATCH 3/5] fix int64 attr --- attr.go | 8 +++++++- internal/internal.proto | 2 +- pql/parser.go | 2 +- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/attr.go b/attr.go index a9eab184c..30365769c 100644 --- a/attr.go +++ b/attr.go @@ -108,8 +108,14 @@ func (s *AttrStore) SetAttrs(id uint64, m map[string]interface{}) error { for k, v := range m { if v == nil { delete(attr, k) - } else { + continue + } + + switch v := v.(type) { + case string, uint64, bool: attr[k] = v + default: + return fmt.Errorf("invalid attr type: %T", v) } } diff --git a/internal/internal.proto b/internal/internal.proto index 31f8688aa..3fe5b6b83 100644 --- a/internal/internal.proto +++ b/internal/internal.proto @@ -28,7 +28,7 @@ message Profile { message Attr { required string Key = 1; optional string StringValue = 2; - optional uint64 UintValue = 3; + optional uint64 UintValue = 3; optional bool BoolValue = 4; } diff --git a/pql/parser.go b/pql/parser.go index 824467103..5f7025121 100644 --- a/pql/parser.go +++ b/pql/parser.go @@ -390,7 +390,7 @@ func (p *Parser) parseSetBitmapAttrsCall() (*SetBitmapAttrs, error) { case string, bool: c.Attrs[key] = v case uint64: - c.Attrs[key] = int64(v) + c.Attrs[key] = v default: return nil, parseErrorf(pos, "invalid SetBitmapAttrs() arg: %v", arg.key) } From a74d826b41b8725b65f055a0bbc5730ba011cf25 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 17 Feb 2016 13:23:30 -0600 Subject: [PATCH 4/5] removed debugging code' --- attr.go | 3 --- fragment.go | 5 ----- 2 files changed, 8 deletions(-) diff --git a/attr.go b/attr.go index 30365769c..0a2a0b5d2 100644 --- a/attr.go +++ b/attr.go @@ -182,13 +182,10 @@ func encodeAttr(key string, value interface{}) *internal.Attr { case string: pb.StringValue = proto.String(value) case float64: - fmt.Println("A") pb.UintValue = proto.Uint64(uint64(value)) case uint64: - fmt.Println("b") pb.UintValue = proto.Uint64(value) case int64: - fmt.Println("c") pb.UintValue = proto.Uint64(uint64(value)) case bool: pb.BoolValue = proto.Bool(value) diff --git a/fragment.go b/fragment.go index 6ce30dbaa..176585ddf 100644 --- a/fragment.go +++ b/fragment.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "log" "os" - "reflect" "sort" "strings" "sync" @@ -385,8 +384,6 @@ func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interfac filters = make(map[interface{}]struct{}) for _, v := range fieldValues { filters[v] = struct{}{} - fmt.Println("B:", reflect.TypeOf(v)) - } } @@ -403,7 +400,6 @@ func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interfac // Apply filter, if set. if filters != nil { attr, err := f.BitmapAttrStore.Attrs(bitmapID) - fmt.Println("C:", reflect.TypeOf(attr), attr) if err != nil { return nil, err } else if attr == nil { @@ -411,7 +407,6 @@ func (f *Fragment) TopN(n int, src *Bitmap, field string, fieldValues []interfac } else if attrValue := attr[field]; attrValue == nil { continue } else if _, ok := filters[attrValue]; !ok { - fmt.Println("A:", reflect.TypeOf(attrValue)) continue } } From 7d6a8846b519cbb6b14a2951cc91f600aace98de Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Mon, 22 Feb 2016 11:30:06 -0600 Subject: [PATCH 5/5] cleaned up tests for SetBit --- fragment_test.go | 26 +++++++++++++------------- handler_test.go | 4 ++-- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/fragment_test.go b/fragment_test.go index ce2fc50f7..441c254e5 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -19,11 +19,11 @@ func TestFragment_SetBit(t *testing.T) { defer f.Close() // Set bits on the fragment. - if err := f.SetBit(120, 1); err != nil { + if _, err := f.SetBit(120, 1); err != nil { t.Fatal(err) - } else if err := f.SetBit(120, 6); err != nil { + } else if _, err := f.SetBit(120, 6); err != nil { t.Fatal(err) - } else if err := f.SetBit(121, 0); err != nil { + } else if _, err := f.SetBit(121, 0); err != nil { t.Fatal(err) } @@ -50,11 +50,11 @@ func TestFragment_ClearBit(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if err := f.SetBit(1000, 1); err != nil { + if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.SetBit(1000, 2); err != nil { t.Fatal(err) - } else if err := f.ClearBit(1000, 1); err != nil { + } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) } @@ -77,11 +77,11 @@ func TestFragment_Snapshot(t *testing.T) { defer f.Close() // Set and then clear bits on the fragment. - if err := f.SetBit(1000, 1); err != nil { + if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) - } else if err := f.SetBit(1000, 2); err != nil { + } else if _, err := f.SetBit(1000, 2); err != nil { t.Fatal(err) - } else if err := f.ClearBit(1000, 1); err != nil { + } else if _, err := f.ClearBit(1000, 1); err != nil { t.Fatal(err) } @@ -218,7 +218,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if err := f.SetBit(i, 0); err != nil { + if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) } } @@ -250,7 +250,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { - if err := f.SetBit(i, 0); err != nil { + if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) } } @@ -332,7 +332,7 @@ func (f *Fragment) Reopen() error { // MustSetBits sets bits on a bitmap. Panic on error. func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { for _, profileID := range profileIDs { - if err := f.SetBit(bitmapID, profileID); err != nil { + if _, err := f.SetBit(bitmapID, profileID); err != nil { panic(err) } } @@ -341,7 +341,7 @@ func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { // MustClearBits clears bits on a bitmap. Panic on error. func (f *Fragment) MustClearBits(bitmapID uint64, profileIDs ...uint64) { for _, profileID := range profileIDs { - if err := f.ClearBit(bitmapID, profileID); err != nil { + if _, err := f.ClearBit(bitmapID, profileID); err != nil { panic(err) } } diff --git a/handler_test.go b/handler_test.go index ebb9cf78b..a8c3a64f8 100644 --- a/handler_test.go +++ b/handler_test.go @@ -210,7 +210,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].GetKey(), attrs[1].GetIntValue(); k != "c" || v != int64(1) { + } else if k, v := attrs[1].GetKey(), attrs[1].GetUintValue(); k != "c" || v != uint64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) } else if k, v := attrs[2].GetKey(), attrs[2].GetBoolValue(); k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v) @@ -267,7 +267,7 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].GetKey(), attrs[0].GetStringValue(); k != "a" || v != "b" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) - } else if k, v := attrs[1].GetKey(), attrs[1].GetIntValue(); k != "c" || v != int64(1) { + } else if k, v := attrs[1].GetKey(), attrs[1].GetUintValue(); k != "c" || v != uint64(1) { t.Fatalf("unexpected attr[1]: %s=%v", k, v) } else if k, v := attrs[2].GetKey(), attrs[2].GetBoolValue(); k != "d" || v != true { t.Fatalf("unexpected attr[2]: %s=%v", k, v)