scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-07 03:17:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #919 from codysoyland/875-value-importer-bug

Browse source 
Fix field value import: Use signed int and respect field minimum
This commit is contained in:
Cody Soyland 2017-10-31 14:37:24 -05:00 • committed by GitHub
commit aaee9b202a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 28 additions and 22 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
client.go
View file

@ -1131,7 +1131,7 @@ func (p Bits) GroupBySlice() map[uint64][]Bit {
// range-encoded frame.
type FieldValue struct {
ColumnID uint64
Value uint64
Value int64
}
// FieldValues represents a slice of field values.
@ -1154,8 +1154,8 @@ func (p FieldValues) ColumnIDs() []uint64 {
}
// Values returns a slice of all the values.
func (p FieldValues) Values() []uint64 {
other := make([]uint64, len(p))
func (p FieldValues) Values() []int64 {
other := make([]int64, len(p))
for i := range p {
other[i] = p[i].Value
}

6
client_test.go
View file

@ -299,7 +299,7 @@ func TestClient_ImportValue(t *testing.T) {
fld := pilosa.Field{
Name: "fld",
Type: pilosa.FieldTypeInt,
Min: 0,
Min: -100,
Max: 100,
}
@ -320,7 +320,7 @@ func TestClient_ImportValue(t *testing.T) {
// Send import request.
c := test.MustNewClient(s.Host())
if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{
{ColumnID: 1, Value: 10},
{ColumnID: 1, Value: -10},
{ColumnID: 2, Value: 20},
{ColumnID: 3, Value: 40},
}); err != nil {
@ -333,7 +333,7 @@ func TestClient_ImportValue(t *testing.T) {
}
// Verify data.
if sum != 70 || cnt != 3 {
if sum != 50 || cnt != 3 {
t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt)
}
}

2
ctl/import.go
View file

@ -293,7 +293,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
val.ColumnID = columnID
// Parse field value.
value, err := strconv.ParseUint(record[1], 10, 64)
value, err := strconv.ParseInt(record[1], 10, 64)
if err != nil {
return fmt.Errorf("invalid value on row %d: %q", rnum, record[1])
}

9
frame.go
View file

@ -882,7 +882,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
}
// ImportValue bulk imports range-encoded value data.
func (f *Frame) ImportValue(fieldName string, columnIDs, values []uint64) error {
func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error {
// Verify that this frame is range-encoded.
if !f.RangeEnabled() {
return fmt.Errorf("Frame not RangeEnabled: %s", f.name)
@ -930,7 +930,12 @@ func (f *Frame) ImportValue(fieldName string, columnIDs, values []uint64) error
return err
}
if err := frag.ImportValue(data.ColumnIDs, data.Values, field.BitDepth()); err != nil {
baseValues := make([]uint64, len(data.Values))
for i, value := range data.Values {
baseValues[i] = uint64(value - field.Min)
}
if err := frag.ImportValue(data.ColumnIDs, baseValues, field.BitDepth()); err != nil {
return err
}
}

2
index.go
View file

@ -668,7 +668,7 @@ type importData struct {
type importValueData struct {
ColumnIDs []uint64
Values []uint64
Values []int64
}
// CreateInputDefinition creates a new input definition.

23
internal/public.pb.go
View file

@ -440,7 +440,7 @@ type ImportValueRequest struct {
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
Field string `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
Values []uint64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"`
Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"`
}
func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} }
@ -483,7 +483,7 @@ func (m *ImportValueRequest) GetColumnIDs() []uint64 {
return nil
}
func (m *ImportValueRequest) GetValues() []uint64 {
func (m *ImportValueRequest) GetValues() []int64 {
if m != nil {
return m.Values
}
@ -1100,7 +1100,8 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) {
if len(m.Values) > 0 {
dAtA16 := make([]byte, len(m.Values)*10)
var j15 int
for _, num := range m.Values {
for _, num1 := range m.Values {
num := uint64(num1)
for num >= 1<<7 {
dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -3244,7 +3245,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
}
case 6:
if wireType == 0 {
var v uint64
var v int64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -3254,7 +3255,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
v |= (uint64(b) & 0x7F) << shift
v |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -3284,7 +3285,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
return io.ErrUnexpectedEOF
}
for iNdEx < postIndex {
var v uint64
var v int64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -3294,7 +3295,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
v |= (uint64(b) & 0x7F) << shift
v |= (int64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -3433,7 +3434,7 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 653 bytes of a gzipped FileDescriptorProto
// 651 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcb, 0x6e, 0xd3, 0x40,
0x14, 0x65, 0x62, 0xe7, 0x75, 0x93, 0x56, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0x14, 0x59, 0x2c, 0xbc,
0x4a, 0xa5, 0xf0, 0x01, 0x08, 0xb7, 0xa9, 0x64, 0x21, 0x2a, 0x98, 0x14, 0xf6, 0x6e, 0x3b, 0x2a,
@ -3472,7 +3473,7 @@ var fileDescriptorPublic = []byte{
0x67, 0x5c, 0x79, 0xae, 0x51, 0x8e, 0x41, 0xa8, 0x73, 0x7b, 0xc5, 0x95, 0xd7, 0x27, 0x53, 0x4b,
0xa0, 0xce, 0xb7, 0x67, 0x8c, 0xda, 0x70, 0x02, 0x47, 0x74, 0x18, 0xff, 0x23, 0x03, 0x6e, 0x2a,
0x25, 0xdd, 0xff, 0xbf, 0x72, 0xd1, 0x37, 0x91, 0xa9, 0x19, 0x25, 0xfa, 0x22, 0xf8, 0x4b, 0xb1,
0x87, 0x30, 0xa0, 0x2a, 0x4c, 0xa1, 0xae, 0x68, 0x50, 0x78, 0xf0, 0x6d, 0x33, 0x63, 0xdf, 0x37,
0x33, 0xf6, 0x63, 0x33, 0x63, 0x1f, 0x7e, 0xce, 0xee, 0x5c, 0x0c, 0xe8, 0x07, 0xfd, 0xf8, 0x57,
0x00, 0x00, 0x00, 0xff, 0xff, 0xf6, 0x73, 0x96, 0xb9, 0xb0, 0x05, 0x00, 0x00,
0x87, 0x30, 0xa0, 0x2a, 0x6c, 0xa1, 0x0d, 0x0a, 0x0f, 0xbe, 0x6d, 0x66, 0xec, 0xfb, 0x66, 0xc6,
0x7e, 0x6c, 0x66, 0xec, 0xc3, 0xcf, 0xd9, 0x9d, 0x8b, 0x01, 0xfd, 0xa0, 0x1f, 0xff, 0x0a, 0x00,
0x00, 0xff, 0xff, 0x4d, 0x1e, 0xdf, 0xba, 0xb0, 0x05, 0x00, 0x00,
}

2
internal/public.proto
View file

@ -79,5 +79,5 @@ message ImportValueRequest {
uint64 Slice = 3;
string Field = 4;
repeated uint64 ColumnIDs = 5;
repeated uint64 Values = 6;
repeated int64 Values = 6;
}