From 27748259a2d2b49e797796c0485b9be65e3f1c83 Mon Sep 17 00:00:00 2001 From: Cody Soyland Date: Tue, 31 Oct 2017 13:32:26 -0500 Subject: [PATCH] Fix field value import: Use signed int and respect field minimum. Fixes #875 --- client.go | 6 +++--- client_test.go | 6 +++--- ctl/import.go | 2 +- frame.go | 9 +++++++-- index.go | 2 +- internal/public.pb.go | 23 ++++++++++++----------- internal/public.proto | 2 +- 7 files changed, 28 insertions(+), 22 deletions(-) diff --git a/client.go b/client.go index 9297f4718..0a893ddf9 100644 --- a/client.go +++ b/client.go @@ -1131,7 +1131,7 @@ func (p Bits) GroupBySlice() map[uint64][]Bit { // range-encoded frame. type FieldValue struct { ColumnID uint64 - Value uint64 + Value int64 } // FieldValues represents a slice of field values. @@ -1154,8 +1154,8 @@ func (p FieldValues) ColumnIDs() []uint64 { } // Values returns a slice of all the values. -func (p FieldValues) Values() []uint64 { - other := make([]uint64, len(p)) +func (p FieldValues) Values() []int64 { + other := make([]int64, len(p)) for i := range p { other[i] = p[i].Value } diff --git a/client_test.go b/client_test.go index 20f601fe7..7e769381b 100644 --- a/client_test.go +++ b/client_test.go @@ -299,7 +299,7 @@ func TestClient_ImportValue(t *testing.T) { fld := pilosa.Field{ Name: "fld", Type: pilosa.FieldTypeInt, - Min: 0, + Min: -100, Max: 100, } @@ -320,7 +320,7 @@ func TestClient_ImportValue(t *testing.T) { // Send import request. c := test.MustNewClient(s.Host()) if err := c.ImportValue(context.Background(), "i", "f", fld.Name, 0, []pilosa.FieldValue{ - {ColumnID: 1, Value: 10}, + {ColumnID: 1, Value: -10}, {ColumnID: 2, Value: 20}, {ColumnID: 3, Value: 40}, }); err != nil { @@ -333,7 +333,7 @@ func TestClient_ImportValue(t *testing.T) { } // Verify data. - if sum != 70 || cnt != 3 { + if sum != 50 || cnt != 3 { t.Fatalf("unexpected values: got sum=%v, count=%v; expected sum=70, cnt=3", sum, cnt) } } diff --git a/ctl/import.go b/ctl/import.go index d2f888956..179385db5 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -293,7 +293,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er val.ColumnID = columnID // Parse field value. - value, err := strconv.ParseUint(record[1], 10, 64) + value, err := strconv.ParseInt(record[1], 10, 64) if err != nil { return fmt.Errorf("invalid value on row %d: %q", rnum, record[1]) } diff --git a/frame.go b/frame.go index 21f160a64..3da55cc93 100644 --- a/frame.go +++ b/frame.go @@ -882,7 +882,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro } // ImportValue bulk imports range-encoded value data. -func (f *Frame) ImportValue(fieldName string, columnIDs, values []uint64) error { +func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { // Verify that this frame is range-encoded. if !f.RangeEnabled() { return fmt.Errorf("Frame not RangeEnabled: %s", f.name) @@ -930,7 +930,12 @@ func (f *Frame) ImportValue(fieldName string, columnIDs, values []uint64) error return err } - if err := frag.ImportValue(data.ColumnIDs, data.Values, field.BitDepth()); err != nil { + baseValues := make([]uint64, len(data.Values)) + for i, value := range data.Values { + baseValues[i] = uint64(value - field.Min) + } + + if err := frag.ImportValue(data.ColumnIDs, baseValues, field.BitDepth()); err != nil { return err } } diff --git a/index.go b/index.go index 3e6cb2518..2031ec5fe 100644 --- a/index.go +++ b/index.go @@ -668,7 +668,7 @@ type importData struct { type importValueData struct { ColumnIDs []uint64 - Values []uint64 + Values []int64 } // CreateInputDefinition creates a new input definition. diff --git a/internal/public.pb.go b/internal/public.pb.go index 81fb2267b..2b39cf4fb 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -440,7 +440,7 @@ type ImportValueRequest struct { Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` Field string `protobuf:"bytes,4,opt,name=Field,proto3" json:"Field,omitempty"` ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` - Values []uint64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` + Values []int64 `protobuf:"varint,6,rep,packed,name=Values" json:"Values,omitempty"` } func (m *ImportValueRequest) Reset() { *m = ImportValueRequest{} } @@ -483,7 +483,7 @@ func (m *ImportValueRequest) GetColumnIDs() []uint64 { return nil } -func (m *ImportValueRequest) GetValues() []uint64 { +func (m *ImportValueRequest) GetValues() []int64 { if m != nil { return m.Values } @@ -1100,7 +1100,8 @@ func (m *ImportValueRequest) MarshalTo(dAtA []byte) (int, error) { if len(m.Values) > 0 { dAtA16 := make([]byte, len(m.Values)*10) var j15 int - for _, num := range m.Values { + for _, num1 := range m.Values { + num := uint64(num1) for num >= 1<<7 { dAtA16[j15] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -3244,7 +3245,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } case 6: if wireType == 0 { - var v uint64 + var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -3254,7 +3255,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3284,7 +3285,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } for iNdEx < postIndex { - var v uint64 + var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -3294,7 +3295,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= (uint64(b) & 0x7F) << shift + v |= (int64(b) & 0x7F) << shift if b < 0x80 { break } @@ -3433,7 +3434,7 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 653 bytes of a gzipped FileDescriptorProto + // 651 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x54, 0xcb, 0x6e, 0xd3, 0x40, 0x14, 0x65, 0x62, 0xe7, 0x75, 0x93, 0x56, 0xd5, 0x08, 0x8a, 0x85, 0x50, 0x14, 0x59, 0x2c, 0xbc, 0x4a, 0xa5, 0xf0, 0x01, 0x08, 0xb7, 0xa9, 0x64, 0x21, 0x2a, 0x98, 0x14, 0xf6, 0x6e, 0x3b, 0x2a, @@ -3472,7 +3473,7 @@ var fileDescriptorPublic = []byte{ 0x67, 0x5c, 0x79, 0xae, 0x51, 0x8e, 0x41, 0xa8, 0x73, 0x7b, 0xc5, 0x95, 0xd7, 0x27, 0x53, 0x4b, 0xa0, 0xce, 0xb7, 0x67, 0x8c, 0xda, 0x70, 0x02, 0x47, 0x74, 0x18, 0xff, 0x23, 0x03, 0x6e, 0x2a, 0x25, 0xdd, 0xff, 0xbf, 0x72, 0xd1, 0x37, 0x91, 0xa9, 0x19, 0x25, 0xfa, 0x22, 0xf8, 0x4b, 0xb1, - 0x87, 0x30, 0xa0, 0x2a, 0x4c, 0xa1, 0xae, 0x68, 0x50, 0x78, 0xf0, 0x6d, 0x33, 0x63, 0xdf, 0x37, - 0x33, 0xf6, 0x63, 0x33, 0x63, 0x1f, 0x7e, 0xce, 0xee, 0x5c, 0x0c, 0xe8, 0x07, 0xfd, 0xf8, 0x57, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xf6, 0x73, 0x96, 0xb9, 0xb0, 0x05, 0x00, 0x00, + 0x87, 0x30, 0xa0, 0x2a, 0x6c, 0xa1, 0x0d, 0x0a, 0x0f, 0xbe, 0x6d, 0x66, 0xec, 0xfb, 0x66, 0xc6, + 0x7e, 0x6c, 0x66, 0xec, 0xc3, 0xcf, 0xd9, 0x9d, 0x8b, 0x01, 0xfd, 0xa0, 0x1f, 0xff, 0x0a, 0x00, + 0x00, 0xff, 0xff, 0x4d, 0x1e, 0xdf, 0xba, 0xb0, 0x05, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 9a94fd469..47b04d8e7 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -79,5 +79,5 @@ message ImportValueRequest { uint64 Slice = 3; string Field = 4; repeated uint64 ColumnIDs = 5; - repeated uint64 Values = 6; + repeated int64 Values = 6; }