From fb7bf118257371035f4473f3136e8d28df861509 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Wed, 23 May 2018 15:05:22 -0500 Subject: [PATCH 01/13] Bit -> Column migration --- api.go | 12 +- cache.go | 4 +- client.go | 86 +++++------ client_test.go | 90 +++++------ cluster.go | 2 +- cluster_test.go | 2 +- cmd/bench.go | 2 +- cmd/bench_test.go | 4 +- cmd/import.go | 4 +- ctl/bench.go | 2 +- ctl/bench_test.go | 2 +- ctl/import.go | 86 +++++------ executor.go | 12 +- executor_test.go | 176 +++++++++++----------- fragment.go | 228 ++++++++++++++-------------- fragment_test.go | 334 ++++++++++++++++++++--------------------- frame.go | 24 +-- handler.go | 10 +- handler_test.go | 32 ++-- holder.go | 2 +- holder_test.go | 90 +++++------ internal/public.pb.go | 42 +++--- row.go | 40 ++--- row_test.go | 22 +-- server/cluster_test.go | 6 +- server/server_test.go | 40 ++--- stats_test.go | 8 +- test/fragment.go | 10 +- test/frame.go | 2 +- time_test.go | 2 +- view.go | 36 ++--- view_test.go | 8 +- 32 files changed, 710 insertions(+), 710 deletions(-) diff --git a/api.go b/api.go index e053e11fa..374dce635 100644 --- a/api.go +++ b/api.go @@ -101,7 +101,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er execOpts := &ExecOptions{ Remote: req.Remote, ExcludeAttrs: req.ExcludeAttrs, - ExcludeBits: req.ExcludeBits, + ExcludeColumns: req.ExcludeColumns, } results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) if err != nil { @@ -110,7 +110,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er resp.Results = results // Fill column attributes if requested. - if req.ColumnAttrs && !req.ExcludeBits { + if req.ColumnAttrs && !req.ExcludeColumns { // Consolidate all column ids across all calls. var columnIDs []uint64 for _, result := range results { @@ -118,7 +118,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if !ok { continue } - columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) + columnIDs = uint64Slice(columnIDs).merge(bm.Columns()) } // Retrieve column attributes across all calls. @@ -305,7 +305,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin // Wrap writer with a CSV writer. cw := csv.NewWriter(w) - // Iterate over each bit. + // Iterate over each column. if err := f.ForEachBit(func(rowID, columnID uint64) error { return cw.Write([]string{ strconv.FormatUint(rowID, 10), @@ -784,7 +784,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // Import into fragment. err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -803,7 +803,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // Import into fragment. err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } diff --git a/cache.go b/cache.go index 5305cfef8..1d45c04be 100644 --- a/cache.go +++ b/cache.go @@ -168,7 +168,7 @@ func NewRankCache(maxEntries uint32) *RankCache { func (c *RankCache) Add(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() - // Ignore if the bit count is below the threshold. + // Ignore if the column count is below the threshold. if n < c.thresholdValue { return } @@ -469,7 +469,7 @@ type BitmapCache interface { // SimpleCache implements BitmapCache // it is meant to be a short-lived cache for cases where writes are continuing to access -// the same bit within a short time frame (i.e. good for write-heavy loads) +// the same column within a short time frame (i.e. good for write-heavy loads) // A read-heavy use case would cause the cache to get bigger, potentially causing the // node to run out of memory. type SimpleCache struct { diff --git a/client.go b/client.go index 0575b838b..66ba115cc 100644 --- a/client.go +++ b/client.go @@ -279,15 +279,15 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index stri return qresp, nil } -// Import bulk imports bits for a single slice to a host. -func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error { +// Import bulk imports columns for a single slice to a host. +func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, columns []Bit) error { if index == "" { return ErrIndexRequired } else if frame == "" { return ErrFrameRequired } - buf, err := marshalImportPayload(index, frame, slice, bits) + buf, err := marshalImportPayload(index, frame, slice, columns) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -308,15 +308,15 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl return nil } -// ImportK bulk imports bits to a host. -func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, bits []Bit) error { +// ImportK bulk imports columns to a host. +func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error { if index == "" { return ErrIndexRequired } else if frame == "" { return ErrFrameRequired } - buf, err := marshalImportPayloadK(index, frame, bits) + buf, err := marshalImportPayloadK(index, frame, columns) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -350,13 +350,13 @@ func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, } // marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { +func marshalImportPayload(index, frame string, slice uint64, columns []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. - rowIDs := Bits(bits).RowIDs() - columnIDs := Bits(bits).ColumnIDs() - timestamps := Bits(bits).Timestamps() + rowIDs := Columns(columns).RowIDs() + columnIDs := Columns(columns).ColumnIDs() + timestamps := Columns(columns).Timestamps() - // Marshal bits to protobufs. + // Marshal columns to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, Frame: frame, @@ -372,13 +372,13 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte } // marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) { +func marshalImportPayloadK(index, frame string, columns []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. - rowKeys := Bits(bits).RowKeys() - columnKeys := Bits(bits).ColumnKeys() - timestamps := Bits(bits).Timestamps() + rowKeys := Columns(columns).RowKeys() + columnKeys := Columns(columns).ColumnKeys() + timestamps := Columns(columns).Timestamps() - // Marshal bits to protobufs. + // Marshal columns to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, Frame: frame, @@ -465,7 +465,7 @@ func marshalImportValuePayload(index, frame, field string, slice uint64, vals [] columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() - // Marshal bits to protobufs. + // Marshal columns to protobufs. buf, err := proto.Marshal(&internal.ImportValueRequest{ Index: index, Frame: frame, @@ -1140,7 +1140,7 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto return nil } -// Bit represents the location of a single bit. +// Bit represents the location of a single column. type Bit struct { RowID uint64 ColumnID uint64 @@ -1149,13 +1149,13 @@ type Bit struct { Timestamp int64 } -// Bits represents a slice of bits. -type Bits []Bit +// Columns represents a slice of columns. +type Columns []Bit -func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p Bits) Len() int { return len(p) } +func (p Columns) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Columns) Len() int { return len(p) } -func (p Bits) Less(i, j int) bool { +func (p Columns) Less(i, j int) bool { if p[i].RowID == p[j].RowID { if p[i].ColumnID < p[j].ColumnID { return p[i].Timestamp < p[j].Timestamp @@ -1166,7 +1166,7 @@ func (p Bits) Less(i, j int) bool { } // RowIDs returns a slice of all the row IDs. -func (p Bits) RowIDs() []uint64 { +func (p Columns) RowIDs() []uint64 { other := make([]uint64, len(p)) for i := range p { other[i] = p[i].RowID @@ -1175,7 +1175,7 @@ func (p Bits) RowIDs() []uint64 { } // ColumnIDs returns a slice of all the column IDs. -func (p Bits) ColumnIDs() []uint64 { +func (p Columns) ColumnIDs() []uint64 { other := make([]uint64, len(p)) for i := range p { other[i] = p[i].ColumnID @@ -1184,7 +1184,7 @@ func (p Bits) ColumnIDs() []uint64 { } // RowKeys returns a slice of all the row keys. -func (p Bits) RowKeys() []string { +func (p Columns) RowKeys() []string { other := make([]string, len(p)) for i := range p { other[i] = p[i].RowKey @@ -1193,7 +1193,7 @@ func (p Bits) RowKeys() []string { } // ColumnKeys returns a slice of all the column keys. -func (p Bits) ColumnKeys() []string { +func (p Columns) ColumnKeys() []string { other := make([]string, len(p)) for i := range p { other[i] = p[i].ColumnKey @@ -1202,7 +1202,7 @@ func (p Bits) ColumnKeys() []string { } // Timestamps returns a slice of all the timestamps. -func (p Bits) Timestamps() []int64 { +func (p Columns) Timestamps() []int64 { other := make([]int64, len(p)) for i := range p { other[i] = p[i].Timestamp @@ -1210,17 +1210,17 @@ func (p Bits) Timestamps() []int64 { return other } -// GroupBySlice returns a map of bits by slice. -func (p Bits) GroupBySlice() map[uint64][]Bit { +// GroupBySlice returns a map of columns by slice. +func (p Columns) GroupBySlice() map[uint64][]Bit { m := make(map[uint64][]Bit) - for _, bit := range p { - slice := bit.ColumnID / SliceWidth - m[slice] = append(m[slice], bit) + for _, column := range p { + slice := column.ColumnID / SliceWidth + m[slice] = append(m[slice], column) } - for slice, bits := range m { - sort.Sort(Bits(bits)) - m[slice] = bits + for slice, columns := range m { + sort.Sort(Columns(columns)) + m[slice] = columns } return m @@ -1277,12 +1277,12 @@ func (p FieldValues) GroupBySlice() map[uint64][]FieldValue { return m } -// BitsByPos represents a slice of bits sorted by internal position. -type BitsByPos []Bit +// ColumnsByPos represents a slice of columns sorted by internal position. +type ColumnsByPos []Bit -func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p BitsByPos) Len() int { return len(p) } -func (p BitsByPos) Less(i, j int) bool { +func (p ColumnsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p ColumnsByPos) Len() int { return len(p) } +func (p ColumnsByPos) Less(i, j int) bool { p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID) if p0 == p1 { return p[i].Timestamp < p[j].Timestamp @@ -1320,8 +1320,8 @@ type InternalClient interface { FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error - ImportK(ctx context.Context, index, frame string, bits []Bit) error + Import(ctx context.Context, index, frame string, slice uint64, columns []Bit) error + ImportK(ctx context.Context, index, frame string, columns []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error diff --git a/client_test.go b/client_test.go index e2b806412..bec6230ad 100644 --- a/client_test.go +++ b/client_test.go @@ -110,26 +110,26 @@ func TestClient_MultiNode(t *testing.T) { } } - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(100, baseBit0+10) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(4, baseBit0+10, baseBit0+11, baseBit0+12) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(22, baseBit0+1, baseBit0+2, baseBit0+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(1, baseBit1+4) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(21, baseBit2+10) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(100, baseBit2+10) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(99, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(98, baseBit2+10, baseBit2+11) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(22, baseBit2+10, baseBit2+11, baseBit2+12) // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay @@ -232,11 +232,11 @@ func TestClient_Import(t *testing.T) { } // Verify data. - if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) { - t.Fatalf("unexpected bits: %+v", a) + if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { + t.Fatalf("unexpected columns: %+v", a) } - if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) { - t.Fatalf("unexpected bits: %+v", a) + if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { + t.Fatalf("unexpected columns: %+v", a) } } @@ -283,14 +283,14 @@ func TestClient_ImportInverseEnabled(t *testing.T) { } // Verify data. - if a := f.Row(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) { - t.Fatalf("unexpected bits: %+v", a) + if a := f.Row(1).Columns(); !reflect.DeepEqual(a, []uint64{0}) { + t.Fatalf("unexpected columns: %+v", a) } - if a := f.Row(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) { - t.Fatalf("unexpected bits: %+v", a) + if a := f.Row(5).Columns(); !reflect.DeepEqual(a, []uint64{0, 200}) { + t.Fatalf("unexpected columns: %+v", a) } - if a := f.Row(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) { - t.Fatalf("unexpected bits: %+v", a) + if a := f.Row(6).Columns(); !reflect.DeepEqual(a, []uint64{200}) { + t.Fatalf("unexpected columns: %+v", a) } } @@ -375,10 +375,10 @@ func TestClient_BackupRestore(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(100, 1, 2, 3, SliceWidth-1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(100, SliceWidth, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetColumns(100, (5*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(200, 20000) s := test.NewServer() defer s.Close() @@ -403,17 +403,17 @@ func TestClient_BackupRestore(t *testing.T) { } // Verify data. - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { - t.Fatalf("unexpected bits(0): %+v", a) + if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { + t.Fatalf("unexpected columns(0): %+v", a) } - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { - t.Fatalf("unexpected bits(0): %+v", a) + if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { + t.Fatalf("unexpected columns(0): %+v", a) } - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { - t.Fatalf("unexpected bits(0): %+v", a) + if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { + t.Fatalf("unexpected columns(0): %+v", a) } - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { - t.Fatalf("unexpected bits: %+v", a) + if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Columns(); !reflect.DeepEqual(a, []uint64{20000}) { + t.Fatalf("unexpected columns: %+v", a) } } @@ -468,8 +468,8 @@ func TestClient_BackupInverseView(t *testing.T) { } // Verify data. - if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { - t.Fatalf("unexpected bits(0): %+v", a) + if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { + t.Fatalf("unexpected columns(0): %+v", a) } } @@ -479,7 +479,7 @@ func TestClient_BackupInvalidView(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(100, 1, 2, 3, SliceWidth-1) s := test.NewServer() defer s.Close() @@ -502,11 +502,11 @@ func TestClient_FragmentBlocks(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set two bits on blocks 0 & 3. + // Set two columns on blocks 0 & 3. hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) - // Set a bit on a different slice. + // Set a column on a different slice. hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1) s := test.NewServer() diff --git a/cluster.go b/cluster.go index 97afa0eef..049136acc 100644 --- a/cluster.go +++ b/cluster.go @@ -930,7 +930,7 @@ func (c *Cluster) Open() error { // (and now in a state of STARTING) so that it can be put to the correct // cluster state. // TODO: Because the normal code path already sends a NodeJoin event (via - // memberlist), this it a bit redundant in most cases. Perhaps determine + // memberlist), this it a column redundant in most cases. Perhaps determine // that the node has been restarted and don't do this step. msg := &internal.NodeEventMessage{ Event: uint32(NodeJoin), diff --git a/cluster_test.go b/cluster_test.go index b09820ac8..21b46060b 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -476,7 +476,7 @@ func TestCluster_ResizeStates(t *testing.T) { t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) } - // Bits + // Columns // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. node1Frame := node1.Holder.Frame("i", "f") node1View := node1Frame.View("standard") diff --git a/cmd/bench.go b/cmd/bench.go index d4b2b8580..f905b49e6 100644 --- a/cmd/bench.go +++ b/cmd/bench.go @@ -45,7 +45,7 @@ Executes a benchmark for a given operation against the index. flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.") flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.") - flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]") + flags.StringVarP(&Bencher.Op, "operation", "o", "set-column", "Operation to perform: choose from [set-column]") flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.") ctl.SetTLSConfig(flags, &Bencher.TLS.CertificatePath, &Bencher.TLS.CertificateKeyPath, &Bencher.TLS.SkipVerify) diff --git a/cmd/bench_test.go b/cmd/bench_test.go index 4b94d9392..c59aae131 100644 --- a/cmd/bench_test.go +++ b/cmd/bench_test.go @@ -33,7 +33,7 @@ func TestBenchHelp(t *testing.T) { func TestBenchConfig(t *testing.T) { tests := []commandTest{ { - args: []string{"bench", "--operation", "set-bit"}, + args: []string{"bench", "--operation", "set-column"}, env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` index = "myindex" @@ -44,7 +44,7 @@ frame = "f1" v.Check(cmd.Bencher.Host, "localhost:12345") v.Check(cmd.Bencher.Index, "myindex") v.Check(cmd.Bencher.Frame, "f1") - v.Check(cmd.Bencher.Op, "set-bit") + v.Check(cmd.Bencher.Op, "set-column") v.Check(cmd.Bencher.N, 0) return v.Error() }, diff --git a/cmd/import.go b/cmd/import.go index fe01738ad..fdf3e1e92 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command importCmd := &cobra.Command{ Use: "import", Short: "Bulk load data into pilosa.", - Long: `Bulk imports one or more CSV files to a host's index and frame. The bits + Long: `Bulk imports one or more CSV files to a host's index and frame. The columns of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: @@ -57,7 +57,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.") flags.StringVarP(&Importer.Field, "field", "", "", "Field to import into.") flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.") - flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") + flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of columns to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") diff --git a/ctl/bench.go b/ctl/bench.go index 4a71169d9..50a18396a 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -62,7 +62,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error { } switch cmd.Op { - case "set-bit": + case "set-column": return cmd.runSetBit(ctx, client) case "": return errors.New("op required") diff --git a/ctl/bench_test.go b/ctl/bench_test.go index 4790ccb44..ac7fc3ad1 100644 --- a/ctl/bench_test.go +++ b/ctl/bench_test.go @@ -56,7 +56,7 @@ func TestBenchCommand_Run(t *testing.T) { r, w, _ := os.Pipe() cm := NewBenchCommand(stdin, w, w) - cm.Op = "set-bit" + cm.Op = "set-column" cm.Host = "localhost:10101" err := cm.Run(context.Background()) diff --git a/ctl/import.go b/ctl/import.go index 004e47e96..20f6a32b9 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -107,7 +107,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { // Import each path and import by slice. for _, path := range cmd.Paths { - // Parse path into bits. + // Parse path into columns. logger.Printf("parsing: %s", path) if err := cmd.importPath(ctx, path); err != nil { return err @@ -129,22 +129,22 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { return nil } -// importPath parses a path into bits and imports it to the server. +// importPath parses a path into columns and imports it to the server. func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { // If a field is provided, treat the import data as values to be range-encoded. if cmd.Field != "" { return cmd.bufferFieldValues(ctx, path) } else { if cmd.StringKeys { - return cmd.bufferBitsK(ctx, path) + return cmd.bufferColumnsK(ctx, path) } else { - return cmd.bufferBits(ctx, path) + return cmd.bufferColumns(ctx, path) } } } -// bufferBits buffers slices of bits to be imported as a batch. -func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { +// bufferColumns buffers slices of columns to be imported as a batch. +func (cmd *ImportCommand) bufferColumns(ctx context.Context, path string) error { a := make([]pilosa.Bit, 0, cmd.BufferSize) var r *csv.Reader @@ -157,7 +157,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { } defer f.Close() - // Read rows as bits. + // Read rows as columns. r = csv.NewReader(f) } else { r = csv.NewReader(cmd.Stdin) @@ -183,21 +183,21 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record)) } - var bit pilosa.Bit + var column pilosa.Bit // Parse row id. rowID, err := strconv.ParseUint(record[0], 10, 64) if err != nil { return fmt.Errorf("invalid row id on row %d: %q", rnum, record[0]) } - bit.RowID = rowID + column.RowID = rowID // Parse column id. columnID, err := strconv.ParseUint(record[1], 10, 64) if err != nil { return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1]) } - bit.ColumnID = columnID + column.ColumnID = columnID // Parse time, if exists. if len(record) > 2 && record[2] != "" { @@ -205,44 +205,44 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { if err != nil { return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2]) } - bit.Timestamp = t.UnixNano() + column.Timestamp = t.UnixNano() } - a = append(a, bit) + a = append(a, column) - // If we've reached the buffer size then import bits. + // If we've reached the buffer size then import columns. if len(a) == cmd.BufferSize { - if err := cmd.importBits(ctx, a); err != nil { + if err := cmd.importColumns(ctx, a); err != nil { return err } a = a[:0] } } - // If there are still bits in the buffer then flush them. - if err := cmd.importBits(ctx, a); err != nil { + // If there are still columns in the buffer then flush them. + if err := cmd.importColumns(ctx, a); err != nil { return err } return nil } -// importBits sends batches of bits to the server. -func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error { +// importColumns sends batches of columns to the server. +func (cmd *ImportCommand) importColumns(ctx context.Context, columns []pilosa.Bit) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // Group bits by slice. - logger.Printf("grouping %d bits", len(bits)) - bitsBySlice := pilosa.Bits(bits).GroupBySlice() + // Group columns by slice. + logger.Printf("grouping %d columns", len(columns)) + columnsBySlice := pilosa.Columns(columns).GroupBySlice() - // Parse path into bits. - for slice, bits := range bitsBySlice { + // Parse path into columns. + for slice, columns := range columnsBySlice { if cmd.Sort { - sort.Sort(pilosa.BitsByPos(bits)) + sort.Sort(pilosa.ColumnsByPos(columns)) } - logger.Printf("importing slice: %d, n=%d", slice, len(bits)) - if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil { + logger.Printf("importing slice: %d, n=%d", slice, len(columns)) + if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, columns); err != nil { return errors.Wrap(err, "importing") } } @@ -250,8 +250,8 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err return nil } -// bufferBitsK buffers slices of keys to be imported as a batch. -func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error { +// bufferColumnsK buffers slices of keys to be imported as a batch. +func (cmd *ImportCommand) bufferColumnsK(ctx context.Context, path string) error { a := make([]pilosa.Bit, 0, cmd.BufferSize) var r *csv.Reader @@ -264,7 +264,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error { } defer f.Close() - // Read rows as bits. + // Read rows as columns. r = csv.NewReader(f) } else { r = csv.NewReader(cmd.Stdin) @@ -290,19 +290,19 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error { return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record)) } - var bit pilosa.Bit + var column pilosa.Bit // Parse row key. if record[0] == "" { return fmt.Errorf("invalid row key on row %d: %q", rnum, record[0]) } - bit.RowKey = record[0] + column.RowKey = record[0] // Parse column key. if record[1] == "" { return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1]) } - bit.ColumnKey = record[1] + column.ColumnKey = record[1] // Parse time, if exists. if len(record) > 2 && record[2] != "" { @@ -310,36 +310,36 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error { if err != nil { return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2]) } - bit.Timestamp = t.UnixNano() + column.Timestamp = t.UnixNano() } - a = append(a, bit) + a = append(a, column) - // If we've reached the buffer size then import bits. + // If we've reached the buffer size then import columns. if len(a) == cmd.BufferSize { - if err := cmd.importBitsK(ctx, a); err != nil { + if err := cmd.importColumnsK(ctx, a); err != nil { return err } a = a[:0] } } - // If there are still bitKs in the buffer then flush them. - if err := cmd.importBitsK(ctx, a); err != nil { + // If there are still columnKs in the buffer then flush them. + if err := cmd.importColumnsK(ctx, a); err != nil { return err } return nil } -// importBitsK sends batches of bitKs to the server. -func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) error { +// importColumnsK sends batches of columnKs to the server. +func (cmd *ImportCommand) importColumnsK(ctx context.Context, columns []pilosa.Bit) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // TODO: does it help to sort the rowKeys? - logger.Printf("importing keys: n=%d", len(bits)) - if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil { + logger.Printf("importing keys: n=%d", len(columns)) + if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, columns); err != nil { return errors.Wrap(err, "importing keys") } @@ -360,7 +360,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er } defer f.Close() - // Read rows as bits. + // Read rows as columns. r = csv.NewReader(f) } else { r = csv.NewReader(cmd.Stdin) diff --git a/executor.go b/executor.go index 2b9f129e4..be242d161 100644 --- a/executor.go +++ b/executor.go @@ -367,7 +367,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } } - if opt.ExcludeBits { + if opt.ExcludeColumns { row.segments = []RowSegment{} } @@ -1064,7 +1064,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) } - // Clear bits for each view. + // Clear columns for each view. switch view { case ViewStandard: return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt) @@ -1164,7 +1164,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, timestamp = &t } - // Set bits for each view. + // Set columns for each view. switch view { case ViewStandard: return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt) @@ -1319,7 +1319,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { return err } - frame.Stats.Count("SetBitmapAttrs", 1, 1.0) + frame.Stats.Count("SetColumnAttrs", 1, 1.0) // Do not forward call if this is already being forwarded. if opt.Remote { @@ -1404,7 +1404,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil { return nil, err } - frame.Stats.Count("SetBitmapAttrs", 1, 1.0) + frame.Stats.Count("SetColumnAttrs", 1, 1.0) } // Do not forward call if this is already being forwarded. @@ -1702,7 +1702,7 @@ type mapResponse struct { type ExecOptions struct { Remote bool ExcludeAttrs bool - ExcludeBits bool + ExcludeColumns bool } // decodeError returns an error representation of s if s is non-blank. diff --git a/executor_test.go b/executor_test.go index 7948ed2fe..655886bb8 100644 --- a/executor_test.go +++ b/executor_test.go @@ -40,7 +40,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set bits. + // Set columns. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ @@ -54,26 +54,26 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } - // Inhibit bits. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil { + // Inhicolumn columns. + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + t.Fatalf("unexpected columns: %+v", columns) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } - // Inhibit attributes. + // Inhicolumn attributes. if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } @@ -89,7 +89,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set bits. + // Set columns. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ @@ -103,8 +103,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{10, 20}) { + t.Fatalf("unexpected columns: %+v", columns) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } @@ -115,17 +115,17 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { func TestExecutor_Execute_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 3) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 4) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -133,7 +133,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Empty_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 1) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { @@ -145,19 +145,19 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { func TestExecutor_Execute_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -176,18 +176,18 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { func TestExecutor_Execute_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -195,13 +195,13 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Empty_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 0) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -209,18 +209,18 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { func TestExecutor_Execute_Xor(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -228,9 +228,9 @@ func TestExecutor_Execute_Xor(t *testing.T) { func TestExecutor_Execute_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(10, 3) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { @@ -255,7 +255,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatal(err) } else { if !res[0].(bool) { - t.Fatalf("expected bit changed") + t.Fatalf("expected column changed") } } @@ -266,7 +266,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatal(err) } else { if res[0].(bool) { - t.Fatalf("expected bit unchanged") + t.Fatalf("expected column unchanged") } } } @@ -409,7 +409,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer hldr.Close() e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set bits for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two slices. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { @@ -462,7 +462,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set bits for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two slices. hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2) @@ -520,7 +520,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set bits for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two slices. hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) @@ -765,7 +765,7 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Fatal(err) } - // Set bits. + // Set columns. if _, err := e.Execute(context.Background(), "i", test.MustParse(` SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00") SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00") @@ -784,8 +784,8 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Run("Standard", func(t *testing.T) { if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) } }) @@ -793,8 +793,8 @@ func TestExecutor_Execute_Range(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 10}) { + t.Fatalf("unexpected columns: %+v", columns) } }) } @@ -854,7 +854,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("EQ", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -863,28 +863,28 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { // NEQ null if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Bits()) + t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Columns()) } }) t.Run("LT", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -892,7 +892,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("LTE", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -900,7 +900,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("GT", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -908,7 +908,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("GTE", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -916,7 +916,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("BETWEEN", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -925,7 +925,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("FieldNotNull", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -933,7 +933,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("BelowMin", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -941,7 +941,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("AboveMax", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -949,16 +949,16 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("LTAboveMax", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo < 200)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits())) + } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) } }) t.Run("GTBelowMin", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo > -200)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits())) + } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) } }) @@ -999,7 +999,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("unexpected slices: %+v", slices) } - // Set bits in slice 0 & 2. + // Set columns in slice 0 & 2. r := pilosa.NewRow( (0*SliceWidth)+1, (0*SliceWidth)+2, @@ -1013,13 +1013,13 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(10, (1*SliceWidth)+1) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -1047,8 +1047,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetColumns(10, (2*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetColumns(10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { @@ -1058,7 +1058,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { } } -// Ensure a remote query can set bits on multiple nodes. +// Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit(t *testing.T) { c := test.NewCluster(2) c.ReplicaN = 2 @@ -1101,7 +1101,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { t.Fatal(err) } - // Verify that one bit is set on both node's holder. + // Verify that one column is set on both node's holder. if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } @@ -1110,7 +1110,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } } -// Ensure a remote query can set bits on multiple nodes. +// Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { c := test.NewCluster(2) c.ReplicaN = 2 @@ -1155,7 +1155,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { t.Fatal(err) } - // Verify that one bit is set on both node's holder. + // Verify that one column is set on both node's holder. if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } @@ -1216,8 +1216,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetColumns(30, (2*SliceWidth)+1) + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetColumns(30, (4*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { diff --git a/fragment.go b/fragment.go index 492524c0d..9d58b09f9 100644 --- a/fragment.go +++ b/fragment.go @@ -171,7 +171,7 @@ func (f *Fragment) Open() error { // Clear checksums. f.checksums = make(map[int][]byte) - // Read last bit to determine max row. + // Read last column to determine max row. pos := f.storage.Max() f.maxRowID = pos / SliceWidth f.stats.Gauge("rows", float64(f.maxRowID), 1.0) @@ -380,7 +380,7 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R return row } -// SetBit sets a bit for a given column & row within the fragment. +// SetBit sets a column for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() @@ -390,10 +390,10 @@ func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { changed = false - // Determine the position of the bit in the storage. + // Determine the position of the column in the storage. pos, err := f.pos(rowID, columnID) if err != nil { - return false, errors.Wrap(err, "getting bit ops") + return false, errors.Wrap(err, "getting column ops") } // Write to storage. @@ -432,7 +432,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } -// ClearBit clears a bit for a given column & row within the fragment. +// ClearBit clears a column for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { f.mu.Lock() @@ -442,10 +442,10 @@ func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { changed = false - // Determine the position of the bit in the storage. + // Determine the position of the column in the storage. pos, err := f.pos(rowID, columnID) if err != nil { - return false, errors.Wrap(err, "getting bit pos") + return false, errors.Wrap(err, "getting column pos") } // Write to storage. @@ -478,7 +478,7 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } -func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { +func (f *Fragment) column(rowID, columnID uint64) (bool, error) { pos, err := f.pos(rowID, columnID) if err != nil { return false, err @@ -486,22 +486,22 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { return f.storage.Contains(pos), nil } -// FieldValue uses a column of bits to read a multi-bit value. -func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +// FieldValue uses a column of columns to read a multi-column value. +func (f *Fragment) FieldValue(columnID uint64, columnDepth uint) (value uint64, exists bool, err error) { f.mu.Lock() defer f.mu.Unlock() - // If existence bit is unset then ignore remaining bits. - if v, err := f.bit(uint64(bitDepth), columnID); err != nil { - return 0, false, errors.Wrap(err, "getting existence bit") + // If existence column is unset then ignore remaining columns. + if v, err := f.column(uint64(columnDepth), columnID); err != nil { + return 0, false, errors.Wrap(err, "getting existence column") } else if !v { return 0, false, nil } - // Compute other bits into a value. - for i := uint(0); i < bitDepth; i++ { - if v, err := f.bit(uint64(i), columnID); err != nil { - return 0, false, errors.Wrapf(err, "getting value bit %d", i) + // Compute other columns into a value. + for i := uint(0); i < columnDepth; i++ { + if v, err := f.column(uint64(i), columnID); err != nil { + return 0, false, errors.Wrapf(err, "getting value column %d", i) } else if v { value |= (1 << i) } @@ -510,12 +510,12 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi return value, true, nil } -// SetFieldValue uses a column of bits to set a multi-bit value. -func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// SetFieldValue uses a column of columns to set a multi-column value. +func (f *Fragment) SetFieldValue(columnID uint64, columnDepth uint, value uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - for i := uint(0); i < bitDepth; i++ { + for i := uint(0); i < columnDepth; i++ { if value&(1< uint(0); i-- { - ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + for i := columnDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (columndepth-1) to 0 row := f.Row(uint64(ii)) x := consider.Difference(row) @@ -649,9 +649,9 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err // FieldMax returns the max of a given field as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { +func (f *Fragment) FieldMax(filter *Row, columnDepth uint) (max, count uint64, err error) { - consider := f.Row(uint64(bitDepth)) + consider := f.Row(uint64(columnDepth)) if filter != nil { consider = consider.Intersect(filter) } @@ -661,8 +661,8 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err return 0, 0, nil } - for i := bitDepth; i > uint(0); i-- { - ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + for i := columnDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (columndepth-1) to 0 row := f.Row(uint64(ii)) x := row.Intersect(consider) @@ -679,31 +679,31 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err } // FieldRange returns bitmaps with a field value encoding matching the predicate. -func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) FieldRange(op pql.Token, columnDepth uint, predicate uint64) (*Row, error) { switch op { case pql.EQ: - return f.fieldRangeEQ(bitDepth, predicate) + return f.fieldRangeEQ(columnDepth, predicate) case pql.NEQ: - return f.fieldRangeNEQ(bitDepth, predicate) + return f.fieldRangeNEQ(columnDepth, predicate) case pql.LT, pql.LTE: - return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE) + return f.fieldRangeLT(columnDepth, predicate, op == pql.LTE) case pql.GT, pql.GTE: - return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE) + return f.fieldRangeGT(columnDepth, predicate, op == pql.GTE) default: return nil, ErrInvalidRangeOperation } } -func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) fieldRangeEQ(columnDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.Row(uint64(columnDepth)) - // Filter any bits that don't match the current bit value. - for i := int(bitDepth - 1); i >= 0; i-- { + // Filter any columns that don't match the current column value. + for i := int(columnDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) - bit := (predicate >> uint(i)) & 1 + column := (predicate >> uint(i)) & 1 - if bit == 1 { + if column == 1 { b = b.Intersect(row) } else { b = b.Difference(row) @@ -713,12 +713,12 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { return b, nil } -func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) fieldRangeNEQ(columnDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.Row(uint64(columnDepth)) // Get the equal bitmap. - eq, err := f.fieldRangeEQ(bitDepth, predicate) + eq, err := f.fieldRangeEQ(columnDepth, predicate) if err != nil { return nil, err } @@ -729,21 +729,21 @@ func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) return b, nil } -func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { +func (f *Fragment) fieldRangeLT(columnDepth uint, predicate uint64, allowEquality bool) (*Row, error) { keep := NewRow() // Start with set of columns with values set. - b := f.Row(uint64(bitDepth)) + b := f.Row(uint64(columnDepth)) - // Filter any bits that don't match the current bit value. + // Filter any columns that don't match the current column value. leadingZeros := true - for i := int(bitDepth - 1); i >= 0; i-- { + for i := int(columnDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) - bit := (predicate >> uint(i)) & 1 + column := (predicate >> uint(i)) & 1 - // Remove any columns with higher bits set. + // Remove any columns with higher columns set. if leadingZeros { - if bit == 0 { + if column == 0 { b = b.Difference(row) continue } else { @@ -751,23 +751,23 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b } } - // Handle last bit differently. - // If bit is zero then return only already kept columns. - // If bit is one then remove any one columns. + // Handle last column differently. + // If column is zero then return only already kept columns. + // If column is one then remove any one columns. if i == 0 && !allowEquality { - if bit == 0 { + if column == 0 { return keep, nil } return b.Difference(row.Difference(keep)), nil } - // If bit is zero then remove all set columns not in excluded bitmap. - if bit == 0 { + // If column is zero then remove all set columns not in excluded bitmap. + if column == 0 { b = b.Difference(row.Difference(keep)) continue } - // If bit is set then add columns for set bits to exclude. + // If column is set then add columns for set columns to exclude. // Don't bother to compute this on the final iteration. if i > 0 { keep = keep.Union(b.Difference(row)) @@ -777,32 +777,32 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b return b, nil } -func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - b := f.Row(uint64(bitDepth)) +func (f *Fragment) fieldRangeGT(columnDepth uint, predicate uint64, allowEquality bool) (*Row, error) { + b := f.Row(uint64(columnDepth)) keep := NewRow() - // Filter any bits that don't match the current bit value. - for i := int(bitDepth - 1); i >= 0; i-- { + // Filter any columns that don't match the current column value. + for i := int(columnDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) - bit := (predicate >> uint(i)) & 1 + column := (predicate >> uint(i)) & 1 - // Handle last bit differently. - // If bit is one then return only already kept columns. - // If bit is zero then remove any unset columns. + // Handle last column differently. + // If column is one then return only already kept columns. + // If column is zero then remove any unset columns. if i == 0 && !allowEquality { - if bit == 1 { + if column == 1 { return keep, nil } return b.Difference(b.Difference(row).Difference(keep)), nil } - // If bit is set then remove all unset columns not already kept. - if bit == 1 { + // If column is set then remove all unset columns not already kept. + if column == 1 { b = b.Difference(b.Difference(row).Difference(keep)) continue } - // If bit is unset then add columns with set bit to keep. + // If column is unset then add columns with set column to keep. // Don't bother to compute this on the final iteration. if i > 0 { keep = keep.Union(b.Intersect(row)) @@ -812,29 +812,29 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b return b, nil } -// FieldNotNull returns the not-null row (stored at bitDepth). -func (f *Fragment) FieldNotNull(bitDepth uint) (*Row, error) { - return f.Row(uint64(bitDepth)), nil +// FieldNotNull returns the not-null row (stored at columnDepth). +func (f *Fragment) FieldNotNull(columnDepth uint) (*Row, error) { + return f.Row(uint64(columnDepth)), nil } // FieldRangeBetween returns bitmaps with a field value encoding matching any value between predicateMin and predicateMax. -func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - b := f.Row(uint64(bitDepth)) +func (f *Fragment) FieldRangeBetween(columnDepth uint, predicateMin, predicateMax uint64) (*Row, error) { + b := f.Row(uint64(columnDepth)) keep1 := NewRow() // GTE keep2 := NewRow() // LTE - // Filter any bits that don't match the current bit value. - for i := int(bitDepth - 1); i >= 0; i-- { + // Filter any columns that don't match the current column value. + for i := int(columnDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) - bit1 := (predicateMin >> uint(i)) & 1 - bit2 := (predicateMax >> uint(i)) & 1 + column1 := (predicateMin >> uint(i)) & 1 + column2 := (predicateMax >> uint(i)) & 1 // GTE predicateMin - // If bit is set then remove all unset columns not already kept. - if bit1 == 1 { + // If column is set then remove all unset columns not already kept. + if column1 == 1 { b = b.Difference(b.Difference(row).Difference(keep1)) } else { - // If bit is unset then add columns with set bit to keep. + // If column is unset then add columns with set column to keep. // Don't bother to compute this on the final iteration. if i > 0 { keep1 = keep1.Union(b.Intersect(row)) @@ -842,11 +842,11 @@ func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax u } // LTE predicateMin - // If bit is zero then remove all set columns not in excluded bitmap. - if bit2 == 0 { + // If column is zero then remove all set columns not in excluded bitmap. + if column2 == 0 { b = b.Difference(row.Difference(keep2)) } else { - // If bit is set then add columns for set bits to exclude. + // If column is set then add columns for set columns to exclude. // Don't bother to compute this on the final iteration. if i > 0 { keep2 = keep2.Union(b.Difference(row)) @@ -867,7 +867,7 @@ func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { return Pos(rowID, columnID), nil } -// ForEachBit executes fn for every bit set in the fragment. +// ForEachBit executes fn for every column set in the fragment. // Errors returned from fn are passed through. func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { f.mu.Lock() @@ -996,13 +996,13 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // If it's too low then don't try finding anymore pairs. threshold := results.Pairs[0].Count - // If the row doesn't have enough bits set before the intersection + // If the row doesn't have enough columns set before the intersection // then we can assume that any remaining rows also have a count too low. if threshold < opt.MinThreshold || cnt < threshold { break } - // Calculate the intersecting bit count and skip if it's below our + // Calculate the intersecting column count and skip if it's below our // last row in our current result set. count := opt.Src.IntersectionCount(f.Row(rowID)) if count < threshold { @@ -1184,7 +1184,7 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i } } -// BlockData returns bits in a block as row & column ID pairs. +// BlockData returns columns in a block as row & column ID pairs. func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() @@ -1196,11 +1196,11 @@ func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { return } -// MergeBlock compares the block's bits and computes a diff with another set of block bits. -// The state of a bit is determined by consensus from all blocks being considered. +// MergeBlock compares the block's columns and computes a diff with another set of block columns. +// The state of a column is determined by consensus from all blocks being considered. // -// For example, if 3 blocks are compared and two have a set bit and one has a -// cleared bit then the bit is considered cleared. The function returns the +// For example, if 3 blocks are compared and two have a set column and one has a +// cleared column then the column is considered cleared. The function returns the // diff per incoming block so that all can be in sync. func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) { // Ensure that all pair sets are of equal length. @@ -1305,14 +1305,14 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e } } - // Set local bits. + // Set local columns. for i := range sets[0].ColumnIDs { if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "setting") } } - // Clear local bits. + // Clear local columns. for i := range clears[0].ColumnIDs { if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") @@ -1322,7 +1322,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e return sets[1:], clears[1:], nil } -// Import bulk imports a set of bits and then snapshots the storage. +// Import bulk imports a set of columns and then snapshots the storage. // This does not affect the fragment's cache. func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { f.mu.Lock() @@ -1335,7 +1335,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { // Disconnect op writer so we don't append updates. f.storage.OpWriter = nil - // Process every bit. + // Process every column. // If an error occurs then reopen the storage. lastID := uint64(0) if err := func() error { @@ -1343,10 +1343,10 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { for i := range rowIDs { rowID, columnID := rowIDs[i], columnIDs[i] - // Determine the position of the bit in the storage. + // Determine the position of the column in the storage. pos, err := f.pos(rowID, columnID) if err != nil { - return errors.Wrap(err, "getting bit pos") + return errors.Wrap(err, "getting column pos") } // Write to storage. @@ -1393,7 +1393,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { } // ImportValue bulk imports a set of range-encoded values. -func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error { +func (f *Fragment) ImportValue(columnIDs, values []uint64, columnDepth uint) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of column ids and values. @@ -1408,7 +1408,7 @@ func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error for i := range columnIDs { columnID, value := columnIDs[i], values[i] - _, err := f.importSetFieldValue(columnID, bitDepth, value) + _, err := f.importSetFieldValue(columnID, columnDepth, value) if err != nil { return errors.Wrap(err, "setting") } diff --git a/fragment_test.go b/fragment_test.go index 22c8fe81f..02042e6f5 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -38,12 +38,12 @@ var ( // SliceWidth is a helper reference to use when testing. const SliceWidth = pilosa.SliceWidth -// Ensure a fragment can set a bit and retrieve it. +// Ensure a fragment can set a column and retrieve it. func TestFragment_SetBit(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set bits on the fragment. + // Set columns on the fragment. if _, err := f.SetBit(120, 1); err != nil { t.Fatal(err) } else if _, err := f.SetBit(120, 6); err != nil { @@ -69,12 +69,12 @@ func TestFragment_SetBit(t *testing.T) { } } -// Ensure a fragment can clear a set bit. +// Ensure a fragment can clear a set column. func TestFragment_ClearBit(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set and then clear bits on the fragment. + // Set and then clear columns on the fragment. if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) } else if _, err := f.SetBit(1000, 2); err != nil { @@ -137,7 +137,7 @@ func TestFragment_SetFieldValue(t *testing.T) { t.Fatal("expected change") } - // Overwriting value should overwrite all bits. + // Overwriting value should overwrite all columns. if changed, err := f.SetFieldValue(100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { @@ -176,13 +176,13 @@ func TestFragment_SetFieldValue(t *testing.T) { }) t.Run("QuickCheck", func(t *testing.T) { - if err := quick.Check(func(bitDepth uint, columnN uint64, values []uint64) bool { - // Limit bit depth & maximum values. - bitDepth = (bitDepth % 62) + 1 + if err := quick.Check(func(columnDepth uint, columnN uint64, values []uint64) bool { + // Limit column depth & maximum values. + columnDepth = (columnDepth % 62) + 1 columnN = (columnN % 99) + 1 for i := range values { - values[i] = values[i] % (1 << bitDepth) + values[i] = values[i] % (1 << columnDepth) } f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") @@ -195,18 +195,18 @@ func TestFragment_SetFieldValue(t *testing.T) { m[columnID] = int64(value) - if _, err := f.SetFieldValue(columnID, bitDepth, value); err != nil { + if _, err := f.SetFieldValue(columnID, columnDepth, value); err != nil { t.Fatal(err) } } // Ensure values are set. for columnID, value := range m { - v, exists, err := f.FieldValue(columnID, bitDepth) + v, exists, err := f.FieldValue(columnID, columnDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { - t.Fatalf("value mismatch: column=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v) + t.Fatalf("value mismatch: column=%d, columndepth=%d, value: %d != %d", columnID, columnDepth, value, v) } else if !exists { t.Fatalf("value should exist: column=%d", columnID) } @@ -221,24 +221,24 @@ func TestFragment_SetFieldValue(t *testing.T) { // Ensure a fragment can sum field values. func TestFragment_FieldSum(t *testing.T) { - const bitDepth = 16 + const columnDepth = 16 f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(nil, bitDepth); err != nil { + if sum, n, err := f.FieldSum(nil, columnDepth); err != nil { t.Fatal(err) } else if n != 4 { t.Fatalf("unexpected count: %d", n) @@ -248,7 +248,7 @@ func TestFragment_FieldSum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(pilosa.NewRow(2000, 4000, 5000), bitDepth); err != nil { + if sum, n, err := f.FieldSum(pilosa.NewRow(2000, 4000, 5000), columnDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -260,25 +260,25 @@ func TestFragment_FieldSum(t *testing.T) { // Ensure a fragment can find the min and max of field values. func TestFragment_FieldMinMax(t *testing.T) { - const bitDepth = 16 + const columnDepth = 16 f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(5000, columnDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil { + } else if _, err := f.SetFieldValue(6000, columnDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil { + } else if _, err := f.SetFieldValue(7000, columnDepth, 0); err != nil { t.Fatal(err) } @@ -296,7 +296,7 @@ func TestFragment_FieldMinMax(t *testing.T) { {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil { + if min, cnt, err := f.FieldMin(test.filter, columnDepth); err != nil { t.Fatal(err) } else if min != test.exp { t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) @@ -320,7 +320,7 @@ func TestFragment_FieldMinMax(t *testing.T) { {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil { + if max, cnt, err := f.FieldMax(test.filter, columnDepth); err != nil { t.Fatal(err) } else if max != test.exp { t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) @@ -333,28 +333,28 @@ func TestFragment_FieldMinMax(t *testing.T) { // Ensure a fragment query for matching fields. func TestFragment_FieldRange(t *testing.T) { - const bitDepth = 16 + const columnDepth = 16 t.Run("EQ", func(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil { t.Fatal(err) } // Query for equality. - if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.EQ, columnDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) @@ -363,21 +363,21 @@ func TestFragment_FieldRange(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil { t.Fatal(err) } // Query for inequality. - if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil { + if b, err := f.FieldRange(pql.NEQ, columnDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) @@ -386,46 +386,46 @@ func TestFragment_FieldRange(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.SetFieldValue(3000, columnDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.SetFieldValue(4000, columnDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.SetFieldValue(5000, columnDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.SetFieldValue(6000, columnDepth, 0); err != nil { t.Fatal(err) } - // Query for fields less than (ending with set bit). - if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil { + // Query for fields less than (ending with set column). + if b, err := f.FieldRange(pql.LT, columnDepth, 301); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than (ending with unset bit). - if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil { + // Query for fields less than (ending with unset column). + if b, err := f.FieldRange(pql.LT, columnDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with set bit). - if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil { + // Query for fields less than or equal to (ending with set column). + if b, err := f.FieldRange(pql.LTE, columnDepth, 301); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with unset bit). - if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil { + // Query for fields less than or equal to (ending with unset column). + if b, err := f.FieldRange(pql.LTE, columnDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) @@ -434,46 +434,46 @@ func TestFragment_FieldRange(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.SetFieldValue(3000, columnDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.SetFieldValue(4000, columnDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.SetFieldValue(5000, columnDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.SetFieldValue(6000, columnDepth, 0); err != nil { t.Fatal(err) } - // Query for fields greater than (ending with unset bit). - if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil { + // Query for fields greater than (ending with unset column). + if b, err := f.FieldRange(pql.GT, columnDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set bit). - if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil { + // Query for fields greater than (ending with set column). + if b, err := f.FieldRange(pql.GT, columnDepth, 301); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset bit). - if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil { + // Query for fields greater than or equal to (ending with unset column). + if b, err := f.FieldRange(pql.GTE, columnDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set bit). - if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil { + // Query for fields greater than or equal to (ending with set column). + if b, err := f.FieldRange(pql.GTE, columnDepth, 301); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) @@ -482,46 +482,46 @@ func TestFragment_FieldRange(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { + } else if _, err := f.SetFieldValue(3000, columnDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { + } else if _, err := f.SetFieldValue(4000, columnDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { + } else if _, err := f.SetFieldValue(5000, columnDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { + } else if _, err := f.SetFieldValue(6000, columnDepth, 0); err != nil { t.Fatal(err) } - // Query for fields greater than (ending with unset bit). - if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil { + // Query for fields greater than (ending with unset column). + if b, err := f.FieldRangeBetween(columnDepth, 300, 2817); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set bit). - if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil { + // Query for fields greater than (ending with set column). + if b, err := f.FieldRangeBetween(columnDepth, 301, 2817); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset bit). - if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil { + // Query for fields greater than or equal to (ending with unset column). + if b, err := f.FieldRangeBetween(columnDepth, 301, 2816); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set bit). - if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil { + // Query for fields greater than or equal to (ending with set column). + if b, err := f.FieldRangeBetween(columnDepth, 300, 2816); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) } @@ -531,7 +531,7 @@ func TestFragment_Snapshot(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set and then clear bits on the fragment. + // Set and then clear columns on the fragment. if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) } else if _, err := f.SetBit(1000, 2); err != nil { @@ -555,12 +555,12 @@ func TestFragment_Snapshot(t *testing.T) { } } -// Ensure a fragment can iterate over all bits in order. +// Ensure a fragment can iterate over all columns in order. func TestFragment_ForEachBit(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set bits on the fragment. + // Set columns on the fragment. if _, err := f.SetBit(100, 20); err != nil { t.Fatal(err) } else if _, err := f.SetBit(2, 38); err != nil { @@ -569,7 +569,7 @@ func TestFragment_ForEachBit(t *testing.T) { t.Fatal(err) } - // Iterate over bits. + // Iterate over columns. var result [][2]uint64 if err := f.ForEachBit(func(rowID, columnID uint64) error { result = append(result, [2]uint64{rowID, columnID}) @@ -578,7 +578,7 @@ func TestFragment_ForEachBit(t *testing.T) { t.Fatal(err) } - // Verify bits are correct. + // Verify columns are correct. if !reflect.DeepEqual(result, [][2]uint64{{2, 37}, {2, 38}, {100, 20}}) { t.Fatalf("unexpected result: %#v", result) } @@ -588,10 +588,10 @@ func TestFragment_ForEachBit(t *testing.T) { func TestFragment_Top(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() - // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 200) - f.MustSetBits(101, 1) - f.MustSetBits(102, 1, 2) + // Set columns on the rows 100, 101, & 102. + f.MustSetColumns(100, 1, 3, 200) + f.MustSetColumns(101, 1) + f.MustSetColumns(102, 1, 2) f.RecalculateCache() // Retrieve top rows. @@ -611,10 +611,10 @@ func TestFragment_Top_Filter(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() - // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 200) - f.MustSetBits(101, 1) - f.MustSetBits(102, 1, 2) + // Set columns on the rows 100, 101, & 102. + f.MustSetColumns(100, 1, 3, 200) + f.MustSetColumns(101, 1) + f.MustSetColumns(102, 1, 2) f.RecalculateCache() // Assign attributes. f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) @@ -644,11 +644,11 @@ func TestFragment_TopN_Intersect(t *testing.T) { // Create an intersecting input row. src := pilosa.NewRow(1, 2, 3) - // Set bits on various rows. - f.MustSetBits(100, 1, 10, 11, 12) // one intersection - f.MustSetBits(101, 1, 2, 3, 4) // three intersections - f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections - f.MustSetBits(103, 1000, 1001, 1002) // no intersection + // Set columns on various rows. + f.MustSetColumns(100, 1, 10, 11, 12) // one intersection + f.MustSetColumns(101, 1, 2, 3, 4) // three intersections + f.MustSetColumns(102, 1, 2, 4, 5, 6) // two intersections + f.MustSetColumns(103, 1000, 1001, 1002) // no intersection f.RecalculateCache() // Retrieve top rows. @@ -663,7 +663,7 @@ func TestFragment_TopN_Intersect(t *testing.T) { } } -// Ensure a fragment can return top rows that have many bits set. +// Ensure a fragment can return top rows that have many columns set. func TestFragment_TopN_Intersect_Large(t *testing.T) { if testing.Short() { t.Skip("short mode") @@ -678,10 +678,10 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, ) - // Set bits on rows 0 - 999. Higher rows have higher bit counts. + // Set columns on rows 0 - 999. Higher rows have higher column counts. for i := uint64(0); i < 1000; i++ { for j := uint64(0); j < i; j++ { - f.MustSetBits(i, j) + f.MustSetColumns(i, j) } } f.RecalculateCache() @@ -710,10 +710,10 @@ func TestFragment_TopN_IDs(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() - // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) + // Set columns on various rows. + f.MustSetColumns(100, 1, 2, 3) + f.MustSetColumns(101, 4, 5, 6, 7) + f.MustSetColumns(102, 8, 9, 10, 11, 12) // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { @@ -731,10 +731,10 @@ func TestFragment_TopN_NopCache(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone) defer f.Close() - // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) + // Set columns on various rows. + f.MustSetColumns(100, 1, 2, 3) + f.MustSetColumns(101, 4, 5, 6, 7) + f.MustSetColumns(102, 8, 9, 10, 11, 12) // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { @@ -783,13 +783,13 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } defer f.Close() - // Set bits on various rows. - f.MustSetBits(100, 1, 2, 3) - f.MustSetBits(101, 4, 5, 6, 7) - f.MustSetBits(102, 8, 9, 10, 11, 12) - f.MustSetBits(103, 8, 9, 10, 11, 12, 13) - f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) - f.MustSetBits(105, 10, 11) + // Set columns on various rows. + f.MustSetColumns(100, 1, 2, 3) + f.MustSetColumns(101, 4, 5, 6, 7) + f.MustSetColumns(102, 8, 9, 10, 11, 12) + f.MustSetColumns(103, 8, 9, 10, 11, 12, 13) + f.MustSetColumns(104, 8, 9, 10, 11, 12, 13, 14) + f.MustSetColumns(105, 10, 11) f.RecalculateCache() @@ -816,7 +816,7 @@ func TestFragment_Checksum(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Retrieve checksum and set bits. + // Retrieve checksum and set columns. orig := f.Checksum() if _, err := f.SetBit(1, 200); err != nil { t.Fatal(err) @@ -838,7 +838,7 @@ func TestFragment_Blocks(t *testing.T) { // Retrieve initial checksum. var prev []pilosa.FragmentBlock - // Set first bit. + // Set first column. if _, err := f.SetBit(0, 0); err != nil { t.Fatal(err) } @@ -848,7 +848,7 @@ func TestFragment_Blocks(t *testing.T) { } prev = blocks - // Set bit on different row. + // Set column on different row. if _, err := f.SetBit(20, 0); err != nil { t.Fatal(err) } @@ -858,7 +858,7 @@ func TestFragment_Blocks(t *testing.T) { } prev = blocks - // Set bit on different column. + // Set column on different column. if _, err := f.SetBit(20, 100); err != nil { t.Fatal(err) } @@ -873,7 +873,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set bits on a different block. + // Set columns on a different block. if _, err := f.SetBit(100, 1); err != nil { t.Fatal(err) } @@ -891,7 +891,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU) defer f.Close() - // Set bits on the fragment. + // Set columns on the fragment. for i := uint64(0); i < 1000; i++ { if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) @@ -941,7 +941,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { t.Fatal(err) } - // Set bits on the fragment. + // Set columns on the fragment. for i := uint64(0); i < 1000; i++ { if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) @@ -976,7 +976,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { f0 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f0.Close() - // Set and then clear bits on the fragment. + // Set and then clear columns on the fragment. if _, err := f0.SetBit(1000, 1); err != nil { t.Fatal(err) } else if _, err := f0.SetBit(1000, 2); err != nil { @@ -1011,8 +1011,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify data in other fragment. - if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { - t.Fatalf("unexpected bits: %+v", a) + if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + t.Fatalf("unexpected columns: %+v", a) } // Close and reopen the fragment & verify the data. @@ -1020,8 +1020,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { t.Fatal(err) } else if n := f1.Cache().Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { - t.Fatalf("unexpected bits (reopen): %+v", a) + } else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + t.Fatalf("unexpected columns (reopen): %+v", a) } } @@ -1083,10 +1083,10 @@ func TestFragment_Tanimoto(t *testing.T) { src := pilosa.NewRow(1, 2, 3) - // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 2, 200) - f.MustSetBits(101, 1, 3) - f.MustSetBits(102, 1, 2, 10, 12) + // Set columns on the rows 100, 101, & 102. + f.MustSetColumns(100, 1, 3, 2, 200) + f.MustSetColumns(101, 1, 3) + f.MustSetColumns(102, 1, 2, 10, 12) f.RecalculateCache() if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { @@ -1106,10 +1106,10 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { src := pilosa.NewRow(1, 2, 3) - // Set bits on the rows 100, 101, & 102. - f.MustSetBits(100, 1, 3, 2, 200) - f.MustSetBits(101, 1, 3) - f.MustSetBits(102, 1, 2, 10, 12) + // Set columns on the rows 100, 101, & 102. + f.MustSetColumns(100, 1, 3, 2, 200) + f.MustSetColumns(101, 1, 3) + f.MustSetColumns(102, 1, 2, 10, 12) f.RecalculateCache() if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { @@ -1129,7 +1129,7 @@ func TestFragment_Snapshot_Run(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set bits on the fragment. + // Set columns on the fragment. for i := uint64(1); i < 3; i++ { if _, err := f.SetBit(1000, i); err != nil { t.Fatal(err) diff --git a/frame.go b/frame.go index de9d3527f..4d0cc9e98 100644 --- a/frame.go +++ b/frame.go @@ -587,7 +587,7 @@ func (f *Frame) DeleteView(name string) error { return nil } -// SetBit sets a bit on a view within the frame. +// SetBit sets a column on a view within the frame. func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. if !IsValidView(name) { @@ -600,7 +600,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed return changed, errors.Wrap(err, "creating view") } - // Set non-time bit. + // Set non-time column. if v, err := view.SetBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "setting on view") } else if v { @@ -612,7 +612,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed return changed, nil } - // If a timestamp is specified then set bits across all views for the quantum. + // If a timestamp is specified then set columns across all views for the quantum. for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { view, err := f.CreateViewIfNotExists(subname) if err != nil { @@ -629,7 +629,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed return changed, nil } -// ClearBit clears a bit within the frame. +// ClearBit clears a column within the frame. func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. if !IsValidView(name) { @@ -642,7 +642,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change return changed, errors.Wrap(err, "creating view") } - // Clear non-time bit. + // Clear non-time column. if v, err := view.ClearBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "setting on view") } else if v { @@ -654,7 +654,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change return changed, nil } - // If a timestamp is specified then clear bits across all views for the quantum. + // If a timestamp is specified then clear columns across all views for the quantum. for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { view, err := f.CreateViewIfNotExists(subname) if err != nil { @@ -846,13 +846,13 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro inverse = []string{ViewInverse} } else { standard = ViewsByTime(ViewStandard, *timestamp, q) - // In order to match the logic of `SetBit()`, we want bits + // In order to match the logic of `SetBit()`, we want columns // with timestamps to write to both time and standard views. standard = append(standard, ViewStandard) inverse = ViewsByTime(ViewInverse, *timestamp, q) } - // Attach bit to each standard view. + // Attach column to each standard view. for _, name := range standard { key := importKey{View: name, Slice: columnID / SliceWidth} data := dataByFragment[key] @@ -862,7 +862,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro } if f.inverseEnabled { - // Attach reversed bits to each inverse view. + // Attach reversed columns to each inverse view. for _, name := range inverse { key := importKey{View: name, Slice: rowID / SliceWidth} data := dataByFragment[key] @@ -909,7 +909,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // ImportValue bulk imports range-encoded value data. func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { viewName := ViewFieldPrefix + fieldName - // Get the field so we know bitDepth. + // Get the field so we know columnDepth. field := f.Field(fieldName) if field == nil { return fmt.Errorf("Field does not exist: %s", fieldName) @@ -939,7 +939,7 @@ func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64 for key, data := range dataByFragment { // The view must already exist (i.e. we can't create it) - // because we need to know bitDepth (based on min/max value). + // because we need to know columnDepth (based on min/max value). view, err := f.CreateViewIfNotExists(key.View) if err != nil { return errors.Wrap(err, "creating view") @@ -1064,7 +1064,7 @@ type Field struct { Max int64 `json:"max,omitempty"` } -// BitDepth returns the number of bits required to store a value between min & max. +// BitDepth returns the number of columns required to store a value between min & max. func (f *Field) BitDepth() uint { for i := uint(0); i < 63; i++ { if f.Max-f.Min < (1 << i) { diff --git a/handler.go b/handler.go index a11c5b43f..08e36aeea 100644 --- a/handler.go +++ b/handler.go @@ -82,7 +82,7 @@ func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse") - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeColumns") h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice") h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") @@ -825,7 +825,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { Slices: slices, ColumnAttrs: q.Get("columnAttrs") == "true", ExcludeAttrs: q.Get("excludeAttrs") == "true", - ExcludeBits: q.Get("excludeBits") == "true", + ExcludeColumns: q.Get("excludeColumns") == "true", }, nil } @@ -1183,8 +1183,8 @@ type QueryRequest struct { // Do not return row attributes, if true. ExcludeAttrs bool - // Do not return bits, if true. - ExcludeBits bool + // Do not return columns, if true. + ExcludeColumns bool // If true, indicates that query is part of a larger distributed query. // If false, this request is on the originating node. @@ -1198,7 +1198,7 @@ func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { ColumnAttrs: pb.ColumnAttrs, Remote: pb.Remote, ExcludeAttrs: pb.ExcludeAttrs, - ExcludeBits: pb.ExcludeBits, + ExcludeColumns: pb.ExcludeColumns, } return req diff --git a/handler_test.go b/handler_test.go index b39b207c5..2952f28b0 100644 --- a/handler_test.go +++ b/handler_test.go @@ -194,13 +194,13 @@ func TestHandler_MaxSlices(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetColumns(30, (1*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetColumns(30, (1*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetColumns(30, (3*SliceWidth)+4) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetColumns(40, (0*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetColumns(40, (0*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetColumns(40, (0*SliceWidth)+8) h := test.NewHandler() h.API.Holder = hldr.Holder @@ -419,7 +419,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -452,7 +452,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -484,8 +484,8 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { @@ -541,8 +541,8 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { @@ -1110,9 +1110,9 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { s.Handler.API.Holder = hldr.Holder defer s.Close() - // Set bits in the index. + // Set columns in the index. f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - f0.MustSetBits(100, 1, 2, 3) + f0.MustSetColumns(100, 1, 2, 3) // Begin backing up from slice i/f/0. resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0") @@ -1145,8 +1145,8 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0) if f1 == nil { t.Fatal("fragment x/y/standard/0 not created") - } else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) { - t.Fatalf("unexpected restored bits: %+v", bits) + } else if columns := f1.Row(100).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 3}) { + t.Fatalf("unexpected restored columns: %+v", columns) } } diff --git a/holder.go b/holder.go index a80ba4539..1d6b4f461 100644 --- a/holder.go +++ b/holder.go @@ -466,7 +466,7 @@ func (h *Holder) flushCaches() { // RecalculateCaches recalculates caches on every index in the holder. This is // probably not practical to call in real-world workloads, but makes writing // integration tests much eaiser, since one doesn't have to wait 10 seconds -// after setting bits to get expected response. +// after setting columns to get expected response. func (h *Holder) RecalculateCaches() { for _, index := range h.Indexes() { index.RecalculateCaches() diff --git a/holder_test.go b/holder_test.go index 419cdc3f2..c5b2ad76d 100644 --- a/holder_test.go +++ b/holder_test.go @@ -330,7 +330,7 @@ func TestHolder_DeleteIndex(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Write bits to separate indexes. + // Write columns to separate indexes. f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0) if _, err := f0.SetBit(100, 200); err != nil { t.Fatal(err) @@ -456,29 +456,29 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected bits(%d/0): %+v", i, a) - } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected bits(%d/2): %+v", i, a) - } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/3): %+v", i, a) - } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/120): %+v", i, a) - } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected bits(%d/200): %+v", i, a) + if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) } f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Bits() + a := f.Row(9).Columns() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a) + t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) } - if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) + if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3) - if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { - t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } } @@ -554,29 +554,29 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected bits(%d/0): %+v", i, a) - } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected bits(%d/2): %+v", i, a) - } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/3): %+v", i, a) - } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/120): %+v", i, a) - } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected bits(%d/200): %+v", i, a) + if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) } f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Bits() + a := f.Row(9).Columns() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a) + t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) } - if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) + if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { - t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -597,16 +597,16 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected bits(%d/0): %+v", i, a) - } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected bits(%d/2): %+v", i, a) - } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/3): %+v", i, a) - } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/120): %+v", i, a) - } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected bits(%d/200): %+v", i, a) + if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) } f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) @@ -615,8 +615,8 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { } f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { - t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } } diff --git a/internal/public.pb.go b/internal/public.pb.go index 22a46565b..e0764bff1 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -43,7 +43,7 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type Row struct { - Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"` + Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } @@ -53,9 +53,9 @@ func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{0} } -func (m *Row) GetBits() []uint64 { +func (m *Row) GetColumns() []uint64 { if m != nil { - return m.Bits + return m.Columns } return nil } @@ -272,7 +272,7 @@ type QueryRequest struct { ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` ExcludeAttrs bool `protobuf:"varint,6,opt,name=ExcludeAttrs,proto3" json:"ExcludeAttrs,omitempty"` - ExcludeBits bool `protobuf:"varint,7,opt,name=ExcludeBits,proto3" json:"ExcludeBits,omitempty"` + ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } @@ -315,9 +315,9 @@ func (m *QueryRequest) GetExcludeAttrs() bool { return false } -func (m *QueryRequest) GetExcludeBits() bool { +func (m *QueryRequest) GetExcludeColumns() bool { if m != nil { - return m.ExcludeBits + return m.ExcludeColumns } return false } @@ -575,10 +575,10 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Bits) > 0 { - dAtA2 := make([]byte, len(m.Bits)*10) + if len(m.Columns) > 0 { + dAtA2 := make([]byte, len(m.Columns)*10) var j1 int - for _, num := range m.Bits { + for _, num := range m.Columns { for num >= 1<<7 { dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -808,7 +808,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) + binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64columns(float64(m.FloatValue)))) i += 8 } return i, nil @@ -912,10 +912,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.ExcludeBits { + if m.ExcludeColumns { dAtA[i] = 0x38 i++ - if m.ExcludeBits { + if m.ExcludeColumns { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -1263,9 +1263,9 @@ func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { func (m *Row) Size() (n int) { var l int _ = l - if len(m.Bits) > 0 { + if len(m.Columns) > 0 { l = 0 - for _, e := range m.Bits { + for _, e := range m.Columns { l += sovPublic(uint64(e)) } n += 1 + sovPublic(uint64(l)) + l @@ -1408,7 +1408,7 @@ func (m *QueryRequest) Size() (n int) { if m.ExcludeAttrs { n += 2 } - if m.ExcludeBits { + if m.ExcludeColumns { n += 2 } return n @@ -1615,7 +1615,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { break } } - m.Bits = append(m.Bits, v) + m.Columns = append(m.Columns, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { @@ -1655,10 +1655,10 @@ func (m *Row) Unmarshal(dAtA []byte) error { break } } - m.Bits = append(m.Bits, v) + m.Columns = append(m.Columns, v) } } else { - return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Columns", wireType) } case 2: if wireType != 2 { @@ -2337,7 +2337,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - m.FloatValue = float64(math.Float64frombits(v)) + m.FloatValue = float64(math.Float64fromcolumns(v)) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -2622,7 +2622,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { m.ExcludeAttrs = bool(v != 0) case 7: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ExcludeBits", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ExcludeColumns", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -2639,7 +2639,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.ExcludeBits = bool(v != 0) + m.ExcludeColumns = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) diff --git a/row.go b/row.go index 740caa7d8..61e735626 100644 --- a/row.go +++ b/row.go @@ -22,7 +22,7 @@ import ( "github.com/pilosa/pilosa/roaring" ) -// Row represents a set of bits. +// Row represents a set of columns. type Row struct { segments []RowSegment @@ -31,9 +31,9 @@ type Row struct { } // NewRow returns a new instance of Row. -func NewRow(bits ...uint64) *Row { +func NewRow(columns ...uint64) *Row { r := &Row{} - for _, i := range bits { + for _, i := range columns { r.SetBit(i) } return r @@ -115,7 +115,7 @@ func (r *Row) Xor(other *Row) *Row { return &Row{segments: segments} } -// Union returns the bitwise union of r and other. +// Union returns the columnwise union of r and other. func (r *Row) Union(other *Row) *Row { var segments []RowSegment itr := newMergeSegmentIterator(r.segments, other.segments) @@ -151,12 +151,12 @@ func (r *Row) Difference(other *Row) *Row { return &Row{segments: segments} } -// SetBit sets the i-th bit of the row. +// SetBit sets the i-th column of the row. func (r *Row) SetBit(i uint64) (changed bool) { return r.createSegmentIfNotExists(i / SliceWidth).SetBit(i) } -// ClearBit clears the i-th bit of the row. +// ClearBit clears the i-th column of the row. func (r *Row) ClearBit(i uint64) (changed bool) { s := r.segment(i / SliceWidth) if s == nil { @@ -226,7 +226,7 @@ func (r *Row) DecrementCount(i uint64) { } } -// Count returns the number of set bits in the row. +// Count returns the number of set columns in the row. func (r *Row) Count() uint64 { var n uint64 for i := range r.segments { @@ -239,9 +239,9 @@ func (r *Row) Count() uint64 { func (r *Row) MarshalJSON() ([]byte, error) { var o struct { Attrs map[string]interface{} `json:"attrs"` - Bits []uint64 `json:"bits"` + Columns []uint64 `json:"columns"` } - o.Bits = r.Bits() + o.Columns = r.Columns() o.Attrs = r.Attrs if o.Attrs == nil { @@ -251,11 +251,11 @@ func (r *Row) MarshalJSON() ([]byte, error) { return json.Marshal(&o) } -// Bits returns the bits in r as a slice of ints. -func (r *Row) Bits() []uint64 { +// Columns returns the columns in r as a slice of ints. +func (r *Row) Columns() []uint64 { a := make([]uint64, 0, r.Count()) for i := range r.segments { - a = append(a, r.segments[i].Bits()...) + a = append(a, r.segments[i].Columns()...) } return a } @@ -267,7 +267,7 @@ func encodeRow(r *Row) *internal.Row { } return &internal.Row{ - Bits: r.Bits(), + Columns: r.Columns(), Attrs: encodeAttrs(r.Attrs), } } @@ -280,7 +280,7 @@ func decodeRow(pr *internal.Row) *Row { r := NewRow() r.Attrs = decodeAttrs(pr.Attrs) - for _, v := range pr.Bits { + for _, v := range pr.Columns { r.SetBit(v) } return r @@ -339,7 +339,7 @@ func (s *RowSegment) Intersect(other *RowSegment) *RowSegment { } } -// Union returns the bitwise union of s and other. +// Union returns the columnwise union of s and other. func (s *RowSegment) Union(other *RowSegment) *RowSegment { data := s.data.Union(&other.data) @@ -372,7 +372,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment { } } -// SetBit sets the i-th bit of the row. +// SetBit sets the i-th column of the row. func (s *RowSegment) SetBit(i uint64) (changed bool) { s.ensureWritable() changed, _ = s.data.Add(i) @@ -382,7 +382,7 @@ func (s *RowSegment) SetBit(i uint64) (changed bool) { return changed } -// ClearBit clears the i-th bit of the row. +// ClearBit clears the i-th column of the row. func (s *RowSegment) ClearBit(i uint64) (changed bool) { s.ensureWritable() @@ -398,8 +398,8 @@ func (s *RowSegment) InvalidateCount() { s.n = s.data.Count() } -// Bits returns a list of all bits set in the segment. -func (s *RowSegment) Bits() []uint64 { +// Columns returns a list of all columns set in the segment. +func (s *RowSegment) Columns() []uint64 { a := make([]uint64, 0, s.Count()) itr := s.data.Iterator() for v, eof := itr.Next(); !eof; v, eof = itr.Next() { @@ -408,7 +408,7 @@ func (s *RowSegment) Bits() []uint64 { return a } -// Count returns the number of set bits in the row. +// Count returns the number of set columns in the row. func (s *RowSegment) Count() uint64 { return s.n } // ensureWritable clones the segment if it is pointing to non-writable data. diff --git a/row_test.go b/row_test.go index 61eae8a81..bc1cc0c68 100644 --- a/row_test.go +++ b/row_test.go @@ -47,7 +47,7 @@ func TestRow_Merge(t *testing.T) { if cnt := test.r1.Count(); cnt != test.exp { t.Fatalf("merged count %d is not %d", cnt, test.exp) } - if length := len(test.r1.Bits()); uint64(length) != test.exp { + if length := len(test.r1.Columns()); uint64(length) != test.exp { t.Fatalf("merged length %d is not %d", length, test.exp) } }) @@ -65,15 +65,15 @@ func TestRow_Xor(t *testing.T) { t.Fatalf("Test 1 Count after xor %d != 3\n", res.Count()) } - if !reflect.DeepEqual(res.Bits(), exp) { - t.Fatalf("Test 2 Results %v != expected %v\n", res.Bits(), exp) + if !reflect.DeepEqual(res.Columns(), exp) { + t.Fatalf("Test 2 Results %v != expected %v\n", res.Columns(), exp) } res = r2.Xor(r1) if res.Count() != 3 { t.Fatalf("Test 3 Count after xor %d != 3\n", res.Count()) } - if !reflect.DeepEqual(res.Bits(), exp) { - t.Fatalf("Test 4 Results %v != expected %v\n", res.Bits(), exp) + if !reflect.DeepEqual(res.Columns(), exp) { + t.Fatalf("Test 4 Results %v != expected %v\n", res.Columns(), exp) } } @@ -86,15 +86,15 @@ func TestRow_Union_Segment(t *testing.T) { if res.Count() != 4 { t.Fatalf("Test 1 Count after Union %d != 5\n", res.Count()) } - if !reflect.DeepEqual(res.Bits(), exp) { - t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp) + if !reflect.DeepEqual(res.Columns(), exp) { + t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Columns(), exp) } res = r2.Union(r1) if res.Count() != 4 { t.Fatalf("Test 3 Count after xor %d != 5\n", res.Count()) } - if !reflect.DeepEqual(res.Bits(), exp) { - t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp) + if !reflect.DeepEqual(res.Columns(), exp) { + t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Columns(), exp) } } @@ -107,7 +107,7 @@ func TestRow_Difference_Segment(t *testing.T) { if res.Count() != 2 { t.Fatalf("Test 1 Count after Difference %d != 5\n", res.Count()) } - if !reflect.DeepEqual(res.Bits(), exp) { - t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Bits(), exp) + if !reflect.DeepEqual(res.Columns(), exp) { + t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Columns(), exp) } } diff --git a/server/cluster_test.go b/server/cluster_test.go index dfb8c9b48..e6777d90c 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -464,12 +464,12 @@ func TestClusterResize_RemoveNode(t *testing.T) { // This is an attempt to ensure there is data on both nodes, but is not guaranteed. // TODO: Deterministic node IDs would ensure consistent results - setBits := "" + setColumns := "" for i := 0; i < 20; i++ { - setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) + setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) } - if _, err := m0.Query("i", "", setBits); err != nil { + if _, err := m0.Query("i", "", setColumns); err != nil { t.Fatal(err) } diff --git a/server/server_test.go b/server/server_test.go index b78e8b79f..c7154daee 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -69,7 +69,7 @@ func TestMain_Set_Quick(t *testing.T) { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "bits": columnIDs, + "columns": columnIDs, "attrs": map[string]interface{}{}, }, }, @@ -92,7 +92,7 @@ func TestMain_Set_Quick(t *testing.T) { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "bits": columnIDs, + "columns": columnIDs, "attrs": map[string]interface{}{}, }, }, @@ -132,7 +132,7 @@ func TestMain_SetRowAttrs(t *testing.T) { t.Fatal(err) } - // Set bits on different rows in different frames. + // Set columns on different rows in different frames. if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil { @@ -157,14 +157,14 @@ func TestMain_SetRowAttrs(t *testing.T) { // Query row x/1. if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x/2. if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -175,19 +175,19 @@ func TestMain_SetRowAttrs(t *testing.T) { // Query rows after reopening. if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x/2. if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } } @@ -205,7 +205,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { t.Fatal(err) } - // Set bits on row. + // Set columns on row. if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { @@ -220,7 +220,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { // Query row. if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -231,7 +231,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { // Query row after reopening. if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } } @@ -266,7 +266,7 @@ func TestMain_InverseSlices(t *testing.T) { SetBit(col=1, frame="f", row=2000) SetBit(col=1, frame="f", row=%d) `, 1*pilosa.SliceWidth)); err != nil { - t.Fatal("setting bits:", err) + t.Fatal("setting columns:", err) } time.Sleep(1 * time.Second) @@ -274,12 +274,12 @@ func TestMain_InverseSlices(t *testing.T) { // Query the cluster. if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil { t.Fatal("another bitmap query:", err) - } else if res != fmt.Sprintf(`{"results":[{"attrs":{},"bits":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" { + } else if res != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" { t.Fatalf("unexpected result: %s", res) } } -// Ensure program can set bits on one cluster and then restore to a second cluster. +// Ensure program can set columns on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { mains1 := test.MustRunMainWithCluster(t, 2) m10 := mains1[0] @@ -304,13 +304,13 @@ func TestMain_FrameRestore(t *testing.T) { SetBit(row=1, frame="f", col=600000) SetBit(row=1, frame="f", col=800000) `); err != nil { - t.Fatal("setting bits:", err) + t.Fatal("setting columns:", err) } // Query row on first cluster. if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { t.Fatal("bitmap query:", err) - } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -348,7 +348,7 @@ func TestMain_FrameRestore(t *testing.T) { // Query row on second cluster. if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { t.Fatal("another bitmap query:", err) - } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { t.Fatalf("2unexpected result: %s", res) } } @@ -408,7 +408,7 @@ func TestMain_RecalculateHashes(t *testing.T) { t.Fatal("create frame:", err) } - // Set some bits + // Set some columns data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { @@ -416,7 +416,7 @@ func TestMain_RecalculateHashes(t *testing.T) { } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { - t.Fatal("setting bits:", err) + t.Fatal("setting columns:", err) } // Calculate caches on the first node @@ -440,7 +440,7 @@ func TestMain_RecalculateHashes(t *testing.T) { } } -// SetCommand represents a command to set a bit. +// SetCommand represents a command to set a column. type SetCommand struct { ID uint64 Frame string diff --git a/stats_test.go b/stats_test.go index 0ec6a0a32..bf4ff14db 100644 --- a/stats_test.go +++ b/stats_test.go @@ -78,7 +78,7 @@ func TestMultiStatClient_Expvar(t *testing.T) { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } - // Expvar should ignore earlier set tags from setbit + // Expvar should ignore earlier set tags from setcolumn if hldr.Stats.Tags() != nil { t.Fatalf("unexpected tag") } @@ -146,7 +146,7 @@ func TestStatsCount_Bitmap(t *testing.T) { } } -func TestStatsCount_SetBitmapAttrs(t *testing.T) { +func TestStatsCount_SetColumnAttrs(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -162,8 +162,8 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) { frame.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { - if name != "SetBitmapAttrs" { - t.Errorf("Expected SetBitmapAttrs, Results %s", name) + if name != "SetColumnAttrs" { + t.Errorf("Expected SetColumnAttrs, Results %s", name) } called = true }, diff --git a/test/fragment.go b/test/fragment.go index 39caa8db2..ba5336af8 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -85,9 +85,9 @@ func (f *Fragment) Reopen() error { return nil } -// MustSetBits sets bits on a row. Panic on error. +// MustSetColumns sets columns on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { +func (f *Fragment) MustSetColumns(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { if _, err := f.SetBit(rowID, columnID); err != nil { panic(err) @@ -95,8 +95,8 @@ func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { } } -// MustClearBits clears bits on a row. Panic on error. -func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) { +// MustClearColumns clears columns on a row. Panic on error. +func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { if _, err := f.ClearBit(rowID, columnID); err != nil { panic(err) @@ -126,7 +126,7 @@ func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { s.attrs[id] = m } -// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk. +// GenerateImportFill generates a set of columns pairs that evenly fill a fragment chunk. func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { ipct := int(pct * 100) for i := 0; i < SliceWidth*rowN; i++ { diff --git a/test/frame.go b/test/frame.go index e107b7d85..825fb06ae 100644 --- a/test/frame.go +++ b/test/frame.go @@ -75,7 +75,7 @@ func (f *Frame) Reopen() error { return nil } -// MustSetBit sets a bit on the frame. Panic on error. +// MustSetBit sets a column on the frame. Panic on error. func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { changed, err := f.SetBit(view, rowID, columnID, t) if err != nil { diff --git a/time_test.go b/time_test.go index 233a2ae1a..87354f779 100644 --- a/time_test.go +++ b/time_test.go @@ -65,7 +65,7 @@ func TestViewByTimeUnit(t *testing.T) { }) } -// Ensure all applicable frame names can be generated when mutating a time bit. +// Ensure all applicable frame names can be generated when mutating a time column. func TestViewsByTime(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) diff --git a/view.go b/view.go index e825aa8ef..41a5ea287 100644 --- a/view.go +++ b/view.go @@ -305,7 +305,7 @@ func (v *View) DeleteFragment(slice uint64) error { return nil } -// SetBit sets a bit within the view. +// SetBit sets a column within the view. func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) @@ -315,7 +315,7 @@ func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { return frag.SetBit(rowID, columnID) } -// ClearBit clears a bit within the view. +// ClearBit clears a column within the view. func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) @@ -325,30 +325,30 @@ func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { return frag.ClearBit(rowID, columnID) } -// FieldValue uses a column of bits to read a multi-bit value. -func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { +// FieldValue uses a column of columns to read a multi-column value. +func (v *View) FieldValue(columnID uint64, columnDepth uint) (value uint64, exists bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return value, exists, err } - return frag.FieldValue(columnID, bitDepth) + return frag.FieldValue(columnID, columnDepth) } -// SetFieldValue uses a column of bits to set a multi-bit value. -func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { +// SetFieldValue uses a column of columns to set a multi-column value. +func (v *View) SetFieldValue(columnID uint64, columnDepth uint, value uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.SetFieldValue(columnID, bitDepth, value) + return frag.SetFieldValue(columnID, columnDepth, value) } // FieldSum returns the sum & count of a field. -func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) { +func (v *View) FieldSum(filter *Row, columnDepth uint) (sum, count uint64, err error) { for _, f := range v.Fragments() { - fsum, fcount, err := f.FieldSum(filter, bitDepth) + fsum, fcount, err := f.FieldSum(filter, columnDepth) if err != nil { return sum, count, err } @@ -359,10 +359,10 @@ func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err erro } // FieldMin returns the min and count of a field. -func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) { +func (v *View) FieldMin(filter *Row, columnDepth uint) (min, count uint64, err error) { var minHasValue bool for _, f := range v.Fragments() { - fmin, fcount, err := f.FieldMin(filter, bitDepth) + fmin, fcount, err := f.FieldMin(filter, columnDepth) if err != nil { return min, count, err } @@ -387,9 +387,9 @@ func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err erro } // FieldMax returns the max and count of a field. -func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { +func (v *View) FieldMax(filter *Row, columnDepth uint) (max, count uint64, err error) { for _, f := range v.Fragments() { - fmax, fcount, err := f.FieldMax(filter, bitDepth) + fmax, fcount, err := f.FieldMax(filter, columnDepth) if err != nil { return max, count, err } @@ -402,10 +402,10 @@ func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err erro } // FieldRange returns rows with a field value encoding matching the predicate. -func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { +func (v *View) FieldRange(op pql.Token, columnDepth uint, predicate uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { - other, err := frag.FieldRange(op, bitDepth, predicate) + other, err := frag.FieldRange(op, columnDepth, predicate) if err != nil { return nil, err } @@ -416,10 +416,10 @@ func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, // FieldRangeBetween returns bitmaps with a field value encoding matching any // value between predicateMin and predicateMax. -func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { +func (v *View) FieldRangeBetween(columnDepth uint, predicateMin, predicateMax uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { - other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax) + other, err := frag.FieldRangeBetween(columnDepth, predicateMin, predicateMax) if err != nil { return nil, err } diff --git a/view_test.go b/view_test.go index 7c7f53a26..7c9e901ae 100644 --- a/view_test.go +++ b/view_test.go @@ -72,9 +72,9 @@ func (v *View) Reopen() error { return v.Open() } -// MustSetBits sets bits on a row. Panic on error. +// MustSetColumns sets columns on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { +func (v *View) MustSetColumns(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { if _, err := v.SetBit(rowID, columnID); err != nil { panic(err) @@ -82,8 +82,8 @@ func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { } } -// MustClearBits clears bits on a row. Panic on error. -func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { +// MustClearColumns clears columns on a row. Panic on error. +func (v *View) MustClearColumns(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { if _, err := v.ClearBit(rowID, columnID); err != nil { panic(err) From de4de2ba2e72b5e2f459304bd3ac4105d60c9733 Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 24 May 2018 08:27:18 -0500 Subject: [PATCH 02/13] changed ExcludeAttr -> ExcludeRowAttr for clarity --- api.go | 4 +- executor.go | 6 +- executor_test.go | 2 +- handler.go | 22 ++++---- internal/public.pb.go | 126 +++++++++++++++++++++--------------------- internal/public.proto | 6 +- 6 files changed, 83 insertions(+), 83 deletions(-) diff --git a/api.go b/api.go index 374dce635..7f8270de3 100644 --- a/api.go +++ b/api.go @@ -99,8 +99,8 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return resp, errors.Wrap(err, "parsing") } execOpts := &ExecOptions{ - Remote: req.Remote, - ExcludeAttrs: req.ExcludeAttrs, + Remote: req.Remote, + ExcludeRowAttrs: req.ExcludeRowAttrs, ExcludeColumns: req.ExcludeColumns, } results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) diff --git a/executor.go b/executor.go index be242d161..2d52547c2 100644 --- a/executor.go +++ b/executor.go @@ -336,7 +336,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C // If the row label is used then return bitmap attributes. row, _ := other.(*Row) if c.Name == "Bitmap" { - if opt.ExcludeAttrs { + if opt.ExcludeRowAttrs { row.Attrs = map[string]interface{}{} } else { idx := e.Holder.Index(index) @@ -1700,8 +1700,8 @@ type mapResponse struct { // ExecOptions represents an execution context for a single Execute() call. type ExecOptions struct { - Remote bool - ExcludeAttrs bool + Remote bool + ExcludeRowAttrs bool ExcludeColumns bool } diff --git a/executor_test.go b/executor_test.go index 655886bb8..7cf56856a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -70,7 +70,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { } // Inhicolumn attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { t.Fatalf("unexpected columns: %+v", columns) diff --git a/handler.go b/handler.go index 08e36aeea..e8d4b5ab4 100644 --- a/handler.go +++ b/handler.go @@ -82,7 +82,7 @@ func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse") - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeColumns") + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice") h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") @@ -821,10 +821,10 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { } return &QueryRequest{ - Query: query, - Slices: slices, - ColumnAttrs: q.Get("columnAttrs") == "true", - ExcludeAttrs: q.Get("excludeAttrs") == "true", + Query: query, + Slices: slices, + ColumnAttrs: q.Get("columnAttrs") == "true", + ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", ExcludeColumns: q.Get("excludeColumns") == "true", }, nil } @@ -1181,7 +1181,7 @@ type QueryRequest struct { ColumnAttrs bool // Do not return row attributes, if true. - ExcludeAttrs bool + ExcludeRowAttrs bool // Do not return columns, if true. ExcludeColumns bool @@ -1193,11 +1193,11 @@ type QueryRequest struct { func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { req := &QueryRequest{ - Query: pb.Query, - Slices: pb.Slices, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, - ExcludeAttrs: pb.ExcludeAttrs, + Query: pb.Query, + Slices: pb.Slices, + ColumnAttrs: pb.ColumnAttrs, + Remote: pb.Remote, + ExcludeRowAttrs: pb.ExcludeRowAttrs, ExcludeColumns: pb.ExcludeColumns, } diff --git a/internal/public.pb.go b/internal/public.pb.go index e0764bff1..0dab2c831 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -43,9 +43,9 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type Row struct { - Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` + Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Row) Reset() { *m = Row{} } @@ -267,11 +267,11 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` - ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` - Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` - ExcludeAttrs bool `protobuf:"varint,6,opt,name=ExcludeAttrs,proto3" json:"ExcludeAttrs,omitempty"` + Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` + ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` + Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` + ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` } @@ -308,9 +308,9 @@ func (m *QueryRequest) GetRemote() bool { return false } -func (m *QueryRequest) GetExcludeAttrs() bool { +func (m *QueryRequest) GetExcludeRowAttrs() bool { if m != nil { - return m.ExcludeAttrs + return m.ExcludeRowAttrs } return false } @@ -808,7 +808,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) { if m.FloatValue != 0 { dAtA[i] = 0x31 i++ - binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64columns(float64(m.FloatValue)))) + binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue)))) i += 8 } return i, nil @@ -902,10 +902,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.ExcludeAttrs { + if m.ExcludeRowAttrs { dAtA[i] = 0x30 i++ - if m.ExcludeAttrs { + if m.ExcludeRowAttrs { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -1405,7 +1405,7 @@ func (m *QueryRequest) Size() (n int) { if m.Remote { n += 2 } - if m.ExcludeAttrs { + if m.ExcludeRowAttrs { n += 2 } if m.ExcludeColumns { @@ -2337,7 +2337,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error { } v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:])) iNdEx += 8 - m.FloatValue = float64(math.Float64fromcolumns(v)) + m.FloatValue = float64(math.Float64frombits(v)) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -2602,7 +2602,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { m.Remote = bool(v != 0) case 6: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ExcludeAttrs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ExcludeRowAttrs", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -2619,7 +2619,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.ExcludeAttrs = bool(v != 0) + m.ExcludeRowAttrs = bool(v != 0) case 7: if wireType != 0 { return fmt.Errorf("proto: wrong wireType = %d for field ExcludeColumns", wireType) @@ -3795,50 +3795,50 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 707 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, - 0x14, 0xbe, 0x13, 0x3b, 0x7f, 0x27, 0x49, 0x55, 0x8d, 0xee, 0xed, 0xb5, 0xae, 0xae, 0x82, 0x65, - 0xb1, 0xf0, 0x2a, 0x95, 0xc2, 0x1e, 0x44, 0xfa, 0x23, 0x45, 0x15, 0x15, 0x4c, 0x4a, 0x59, 0xbb, - 0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x08, 0x16, - 0x88, 0x15, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x73, 0xc6, 0x13, 0x3b, 0xad, 0x04, 0x2c, 0xd8, - 0xcd, 0xf7, 0x7d, 0x33, 0xc7, 0xf3, 0xcd, 0xf9, 0x4e, 0x02, 0xc3, 0xac, 0xbc, 0x48, 0xe2, 0xcb, - 0x49, 0x96, 0x2b, 0xad, 0x78, 0x2f, 0x4e, 0xb5, 0xcc, 0xd3, 0x28, 0x09, 0x16, 0xe0, 0x08, 0xb5, - 0xe2, 0x1c, 0xdc, 0x59, 0xac, 0x0b, 0x8f, 0xf9, 0x4e, 0xe8, 0x0a, 0x5a, 0xf3, 0x87, 0xd0, 0x7e, - 0xaa, 0x75, 0x5e, 0x78, 0x2d, 0xdf, 0x09, 0x07, 0xd3, 0x9d, 0x89, 0x3d, 0x34, 0x41, 0x5a, 0x18, - 0x11, 0x4f, 0x9e, 0xc8, 0x75, 0xe1, 0x39, 0xbe, 0x13, 0xf6, 0x05, 0xad, 0x83, 0xc7, 0xe0, 0x3e, - 0x8f, 0xe2, 0x9c, 0xef, 0x40, 0x6b, 0x7e, 0xe8, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd, 0x0f, 0xf9, - 0xdf, 0xd0, 0x3e, 0x50, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x5d, 0x70, 0x4e, 0xe4, 0xda, - 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0xce, 0xa3, 0x64, 0xa3, 0x9e, 0x47, 0x09, - 0x15, 0x71, 0x04, 0x2e, 0xb7, 0xab, 0x38, 0x55, 0x95, 0xe0, 0x25, 0x38, 0xb3, 0x58, 0xa3, 0x28, - 0xd4, 0x6a, 0xf3, 0x55, 0x03, 0xf8, 0x7f, 0xd0, 0x3b, 0x50, 0x49, 0xb9, 0x4c, 0xe7, 0x87, 0xd5, - 0xb7, 0x37, 0x98, 0xff, 0x0f, 0xfd, 0xb3, 0x78, 0x29, 0x0b, 0x1d, 0x2d, 0x33, 0xba, 0x84, 0x23, - 0x6a, 0x22, 0x78, 0x05, 0x23, 0xb3, 0x13, 0xdd, 0x2e, 0xa4, 0xbe, 0xe7, 0xe9, 0xf7, 0x5e, 0xe9, - 0xbe, 0xc7, 0xf7, 0x0c, 0x5c, 0xd4, 0xac, 0xc4, 0x36, 0x12, 0x3e, 0xe9, 0xd9, 0x3a, 0x93, 0xd5, - 0x4d, 0x69, 0xcd, 0x7d, 0x18, 0x2c, 0x74, 0x1e, 0xa7, 0xd7, 0xe7, 0x51, 0x52, 0xca, 0xaa, 0x50, - 0x93, 0x42, 0x8f, 0xf3, 0x54, 0x1b, 0xd9, 0x25, 0x1b, 0x1b, 0x8c, 0x1e, 0x67, 0x4a, 0x25, 0x46, - 0x6c, 0xfb, 0x2c, 0xec, 0x89, 0x9a, 0xe0, 0x63, 0x80, 0xe3, 0x44, 0x45, 0xd5, 0xd9, 0x8e, 0xcf, - 0x42, 0x26, 0x1a, 0x4c, 0xb0, 0x0f, 0x5d, 0xbc, 0xe9, 0xb3, 0x28, 0xab, 0xdd, 0xb2, 0x9f, 0xb8, - 0x0d, 0x3e, 0x30, 0x18, 0xbe, 0x28, 0x65, 0xbe, 0x16, 0xf2, 0x4d, 0x29, 0x0b, 0xea, 0x0a, 0xe1, - 0xca, 0xa5, 0x01, 0x7c, 0x0f, 0x3a, 0x8b, 0x24, 0xbe, 0x94, 0xe6, 0xed, 0x5c, 0x51, 0x21, 0xf4, - 0x5a, 0xbf, 0x79, 0x41, 0x5e, 0x7b, 0xa2, 0x49, 0xe1, 0x49, 0x21, 0x97, 0x4a, 0x5b, 0x33, 0x15, - 0xe2, 0x01, 0x0c, 0x8f, 0x6e, 0x2e, 0x93, 0xf2, 0x4a, 0x9a, 0xa3, 0x1d, 0x52, 0xb7, 0x38, 0xac, - 0x5e, 0x61, 0x4a, 0x7c, 0xd7, 0x54, 0x6f, 0x50, 0xc1, 0x5b, 0x06, 0xa3, 0xea, 0xfa, 0x45, 0xa6, - 0xd2, 0x42, 0x62, 0x8f, 0x8e, 0xf2, 0xdc, 0xf6, 0xe8, 0x28, 0xcf, 0xf9, 0x3e, 0x74, 0x85, 0x2c, - 0xca, 0x44, 0xdb, 0xc6, 0xff, 0x53, 0x3f, 0x85, 0x3d, 0x5b, 0x26, 0x5a, 0xd8, 0x5d, 0xfc, 0x09, - 0xec, 0x6c, 0x05, 0xc9, 0x4c, 0xcc, 0x60, 0xfa, 0x6f, 0x7d, 0x6e, 0x4b, 0x17, 0x77, 0xb6, 0x07, - 0x1f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x03, 0x9a, 0x5c, 0xba, 0xd3, 0x60, 0x3a, 0xaa, 0xab, 0x08, - 0xb5, 0x12, 0x34, 0xd3, 0x43, 0x60, 0xa7, 0x55, 0x86, 0xd8, 0x29, 0x76, 0x0e, 0x67, 0xd2, 0x7e, - 0xb6, 0xd1, 0x39, 0xa4, 0x85, 0x11, 0xb9, 0x07, 0xdd, 0x83, 0xd7, 0x51, 0x7a, 0x2d, 0xaf, 0x28, - 0x43, 0x3d, 0x61, 0x21, 0x9f, 0xd4, 0x33, 0x49, 0x8f, 0x3e, 0x98, 0xf2, 0xba, 0x84, 0x55, 0x44, - 0x3d, 0xb7, 0x36, 0xc4, 0xd8, 0x82, 0x91, 0x09, 0x71, 0xf0, 0x8d, 0xc1, 0x68, 0xbe, 0xcc, 0x54, - 0xae, 0x1b, 0xc1, 0x98, 0xa7, 0x57, 0xf2, 0xc6, 0x06, 0x83, 0x00, 0xb2, 0xc7, 0x79, 0xb4, 0x34, - 0x13, 0xd0, 0x17, 0x06, 0x20, 0x4b, 0x01, 0xa1, 0x40, 0xb8, 0xc2, 0x00, 0x8a, 0x02, 0xce, 0x78, - 0xe1, 0xb9, 0x26, 0x44, 0x06, 0x61, 0xe4, 0xed, 0x88, 0x17, 0x5e, 0x9b, 0xa4, 0x9a, 0xc0, 0xc8, - 0x6f, 0x66, 0x1c, 0x63, 0xe2, 0x84, 0x8e, 0x68, 0x30, 0xf8, 0x0e, 0x42, 0xad, 0xe8, 0x87, 0xad, - 0x4b, 0x3f, 0x6c, 0x16, 0xe2, 0x49, 0x53, 0x86, 0xc4, 0x1e, 0x89, 0x0d, 0x26, 0xf8, 0xc4, 0x80, - 0x1b, 0x8f, 0x34, 0x3c, 0x7f, 0xce, 0x28, 0xee, 0x8d, 0x65, 0x62, 0x1a, 0x83, 0x7b, 0x11, 0xfc, - 0xc2, 0xe6, 0x1e, 0x74, 0xe8, 0x16, 0xd6, 0x62, 0x85, 0xee, 0x98, 0xe8, 0xde, 0x35, 0x31, 0xdb, - 0xfd, 0x7c, 0x3b, 0x66, 0x5f, 0x6e, 0xc7, 0xec, 0xeb, 0xed, 0x98, 0xbd, 0xfb, 0x3e, 0xfe, 0xeb, - 0xa2, 0x43, 0x7f, 0x1c, 0x8f, 0x7e, 0x04, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x35, 0x66, 0x28, 0x48, - 0x06, 0x00, 0x00, + // 709 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c, + 0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c, + 0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba, + 0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4, + 0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd, + 0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64, + 0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8, + 0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82, + 0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69, + 0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f, + 0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd, + 0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e, + 0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e, + 0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62, + 0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf, + 0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8, + 0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77, + 0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e, + 0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35, + 0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88, + 0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59, + 0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7, + 0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c, + 0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd, + 0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85, + 0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3, + 0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18, + 0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3, + 0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4, + 0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96, + 0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60, + 0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec, + 0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd, + 0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e, + 0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c, + 0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41, + 0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62, + 0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9, + 0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b, + 0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82, + 0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8, + 0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd, + 0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7, + 0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7, + 0xa4, 0x5a, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index f2afdb7f9..9207d3a67 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package internal; message Row { - repeated uint64 Bits = 1; + repeated uint64 Columns = 1; repeated string Keys = 3; repeated Attr Attrs = 2; } @@ -49,8 +49,8 @@ message QueryRequest { repeated uint64 Slices = 2; bool ColumnAttrs = 3; bool Remote = 5; - bool ExcludeAttrs = 6; - bool ExcludeBits = 7; + bool ExcludeRowAttrs = 6; + bool ExcludeColumns = 7; } message QueryResponse { From c3166fb0c129d6b421bbc6e628de61852ba940b0 Mon Sep 17 00:00:00 2001 From: Yuce Tekol Date: Thu, 24 May 2018 18:36:09 +0300 Subject: [PATCH 03/13] ClerBit doc fix --- docs/query-language.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/query-language.md b/docs/query-language.md index bbfe1cbb7..a598bf351 100644 --- a/docs/query-language.md +++ b/docs/query-language.md @@ -200,14 +200,15 @@ SetColumnAttrs(col=10, url=null) **Spec:** ``` -ClearBit(, , , - [timestamp=TIMESTAMP]) +ClearBit(, , ) ``` **Description:** `ClearBit` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given frame from the given column. +Note that clearing bits from time views is not supported. + **Result Type:** boolean A return value of `true` indicates that the bit was toggled from 1 to 0. From e3da6efe3c4dcbdc28556b16831bd87d2b125dbb Mon Sep 17 00:00:00 2001 From: Todd Gruben Date: Thu, 24 May 2018 12:48:24 -0500 Subject: [PATCH 04/13] revert to old labels on BSI; revert MustSetColumns --- client.go | 62 +++++++------- client_test.go | 46 +++++------ ctl/import.go | 14 ++-- executor.go | 2 +- executor_test.go | 66 +++++++-------- fragment.go | 174 +++++++++++++++++++-------------------- fragment_test.go | 208 +++++++++++++++++++++++------------------------ handler_test.go | 14 ++-- test/fragment.go | 4 +- test/frame.go | 2 +- time_test.go | 2 +- view_test.go | 4 +- 12 files changed, 299 insertions(+), 299 deletions(-) diff --git a/client.go b/client.go index 66ba115cc..c385c5177 100644 --- a/client.go +++ b/client.go @@ -279,15 +279,15 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index stri return qresp, nil } -// Import bulk imports columns for a single slice to a host. -func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, columns []Bit) error { +// Import bulk imports bits for a single slice to a host. +func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error { if index == "" { return ErrIndexRequired } else if frame == "" { return ErrFrameRequired } - buf, err := marshalImportPayload(index, frame, slice, columns) + buf, err := marshalImportPayload(index, frame, slice, bits) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -350,11 +350,11 @@ func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string, } // marshalImportPayload marshalls the import parameters into a protobuf byte slice. -func marshalImportPayload(index, frame string, slice uint64, columns []Bit) ([]byte, error) { +func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. - rowIDs := Columns(columns).RowIDs() - columnIDs := Columns(columns).ColumnIDs() - timestamps := Columns(columns).Timestamps() + rowIDs := Bits(bits).RowIDs() + columnIDs := Bits(bits).ColumnIDs() + timestamps := Bits(bits).Timestamps() // Marshal columns to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ @@ -372,11 +372,11 @@ func marshalImportPayload(index, frame string, slice uint64, columns []Bit) ([]b } // marshalImportPayloadK marshalls the import parameters into a protobuf byte slice. -func marshalImportPayloadK(index, frame string, columns []Bit) ([]byte, error) { +func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) { // Separate row and column IDs to reduce allocations. - rowKeys := Columns(columns).RowKeys() - columnKeys := Columns(columns).ColumnKeys() - timestamps := Columns(columns).Timestamps() + rowKeys := Bits(bits).RowKeys() + columnKeys := Bits(bits).ColumnKeys() + timestamps := Bits(bits).Timestamps() // Marshal columns to protobufs. buf, err := proto.Marshal(&internal.ImportRequest{ @@ -1140,7 +1140,7 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto return nil } -// Bit represents the location of a single column. +// Bit represents the location of a the intersection of a row and a column. type Bit struct { RowID uint64 ColumnID uint64 @@ -1149,13 +1149,13 @@ type Bit struct { Timestamp int64 } -// Columns represents a slice of columns. -type Columns []Bit +// Bits represents a slice of Bit. +type Bits []Bit -func (p Columns) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p Columns) Len() int { return len(p) } +func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p Bits) Len() int { return len(p) } -func (p Columns) Less(i, j int) bool { +func (p Bits) Less(i, j int) bool { if p[i].RowID == p[j].RowID { if p[i].ColumnID < p[j].ColumnID { return p[i].Timestamp < p[j].Timestamp @@ -1166,7 +1166,7 @@ func (p Columns) Less(i, j int) bool { } // RowIDs returns a slice of all the row IDs. -func (p Columns) RowIDs() []uint64 { +func (p Bits) RowIDs() []uint64 { other := make([]uint64, len(p)) for i := range p { other[i] = p[i].RowID @@ -1175,7 +1175,7 @@ func (p Columns) RowIDs() []uint64 { } // ColumnIDs returns a slice of all the column IDs. -func (p Columns) ColumnIDs() []uint64 { +func (p Bits) ColumnIDs() []uint64 { other := make([]uint64, len(p)) for i := range p { other[i] = p[i].ColumnID @@ -1184,7 +1184,7 @@ func (p Columns) ColumnIDs() []uint64 { } // RowKeys returns a slice of all the row keys. -func (p Columns) RowKeys() []string { +func (p Bits) RowKeys() []string { other := make([]string, len(p)) for i := range p { other[i] = p[i].RowKey @@ -1193,7 +1193,7 @@ func (p Columns) RowKeys() []string { } // ColumnKeys returns a slice of all the column keys. -func (p Columns) ColumnKeys() []string { +func (p Bits) ColumnKeys() []string { other := make([]string, len(p)) for i := range p { other[i] = p[i].ColumnKey @@ -1202,7 +1202,7 @@ func (p Columns) ColumnKeys() []string { } // Timestamps returns a slice of all the timestamps. -func (p Columns) Timestamps() []int64 { +func (p Bits) Timestamps() []int64 { other := make([]int64, len(p)) for i := range p { other[i] = p[i].Timestamp @@ -1211,7 +1211,7 @@ func (p Columns) Timestamps() []int64 { } // GroupBySlice returns a map of columns by slice. -func (p Columns) GroupBySlice() map[uint64][]Bit { +func (p Bits) GroupBySlice() map[uint64][]Bit { m := make(map[uint64][]Bit) for _, column := range p { slice := column.ColumnID / SliceWidth @@ -1219,7 +1219,7 @@ func (p Columns) GroupBySlice() map[uint64][]Bit { } for slice, columns := range m { - sort.Sort(Columns(columns)) + sort.Sort(Bits(columns)) m[slice] = columns } @@ -1277,12 +1277,12 @@ func (p FieldValues) GroupBySlice() map[uint64][]FieldValue { return m } -// ColumnsByPos represents a slice of columns sorted by internal position. -type ColumnsByPos []Bit +// BitsByPos represents a slice of columns sorted by internal position. +type BitsByPos []Bit -func (p ColumnsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } -func (p ColumnsByPos) Len() int { return len(p) } -func (p ColumnsByPos) Less(i, j int) bool { +func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (p BitsByPos) Len() int { return len(p) } +func (p BitsByPos) Less(i, j int) bool { p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID) if p0 == p1 { return p[i].Timestamp < p[j].Timestamp @@ -1320,8 +1320,8 @@ type InternalClient interface { FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error) - Import(ctx context.Context, index, frame string, slice uint64, columns []Bit) error - ImportK(ctx context.Context, index, frame string, columns []Bit) error + Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error + ImportK(ctx context.Context, index, frame string, bits []Bit) error EnsureIndex(ctx context.Context, name string, options IndexOptions) error EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error diff --git a/client_test.go b/client_test.go index bec6230ad..82156d138 100644 --- a/client_test.go +++ b/client_test.go @@ -110,26 +110,26 @@ func TestClient_MultiNode(t *testing.T) { } } - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(100, baseBit0+10) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(4, baseBit0+10, baseBit0+11, baseBit0+12) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) - hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(22, baseBit0+1, baseBit0+2, baseBit0+10) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5) + hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(1, baseBit1+4) - hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4) + hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(21, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(100, baseBit2+10) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(99, baseBit2+10, baseBit2+11, baseBit2+12) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(98, baseBit2+10, baseBit2+11) - hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(22, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11) + hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12) // Rebuild the RankCache. // We have to do this to avoid the 10-second cache invalidation delay @@ -375,10 +375,10 @@ func TestClient_BackupRestore(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(100, 1, 2, 3, SliceWidth-1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(100, SliceWidth, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetColumns(100, (5*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(200, 20000) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) s := test.NewServer() defer s.Close() @@ -479,7 +479,7 @@ func TestClient_BackupInvalidView(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(100, 1, 2, 3, SliceWidth-1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) s := test.NewServer() defer s.Close() diff --git a/ctl/import.go b/ctl/import.go index 20f6a32b9..e193cd070 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -228,21 +228,21 @@ func (cmd *ImportCommand) bufferColumns(ctx context.Context, path string) error } // importColumns sends batches of columns to the server. -func (cmd *ImportCommand) importColumns(ctx context.Context, columns []pilosa.Bit) error { +func (cmd *ImportCommand) importColumns(ctx context.Context, bits []pilosa.Bit) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // Group columns by slice. - logger.Printf("grouping %d columns", len(columns)) - columnsBySlice := pilosa.Columns(columns).GroupBySlice() + logger.Printf("grouping %d columns", len(bits)) + bitsBySlice := pilosa.Bits(bits).GroupBySlice() // Parse path into columns. - for slice, columns := range columnsBySlice { + for slice, chunk := range bitsBySlice { if cmd.Sort { - sort.Sort(pilosa.ColumnsByPos(columns)) + sort.Sort(pilosa.BitsByPos(chunk)) } - logger.Printf("importing slice: %d, n=%d", slice, len(columns)) - if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, columns); err != nil { + logger.Printf("importing slice: %d, n=%d", slice, len(chunk)) + if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil { return errors.Wrap(err, "importing") } } diff --git a/executor.go b/executor.go index 2d52547c2..bdc7618fe 100644 --- a/executor.go +++ b/executor.go @@ -1319,7 +1319,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { return err } - frame.Stats.Count("SetColumnAttrs", 1, 1.0) + frame.Stats.Count("SetRowAttrs", 1, 1.0) // Do not forward call if this is already being forwarded. if opt.Remote { diff --git a/executor_test.go b/executor_test.go index 7cf56856a..3225cc747 100644 --- a/executor_test.go +++ b/executor_test.go @@ -54,8 +54,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { - t.Fatalf("unexpected columns: %+v", columns) + } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } @@ -115,11 +115,11 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { func TestExecutor_Execute_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 4) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -133,7 +133,7 @@ func TestExecutor_Execute_Difference(t *testing.T) { func TestExecutor_Execute_Empty_Difference(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil { @@ -145,13 +145,13 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) { func TestExecutor_Execute_Intersect(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -176,12 +176,12 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) { func TestExecutor_Execute_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -195,7 +195,7 @@ func TestExecutor_Execute_Union(t *testing.T) { func TestExecutor_Execute_Empty_Union(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { @@ -209,12 +209,12 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { func TestExecutor_Execute_Xor(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 0) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2) - hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(11, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2) + hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { @@ -228,9 +228,9 @@ func TestExecutor_Execute_Xor(t *testing.T) { func TestExecutor_Execute_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(10, 3) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2) e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { @@ -1013,7 +1013,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(10, (1*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { @@ -1047,8 +1047,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetColumns(10, (2*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetColumns(10, (2*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil { @@ -1216,8 +1216,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() s.Handler.API.Holder = hldr.Holder - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetColumns(30, (2*SliceWidth)+1) - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetColumns(30, (4*SliceWidth)+2) + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1) + hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2) e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil { diff --git a/fragment.go b/fragment.go index 9d58b09f9..8651f0a4a 100644 --- a/fragment.go +++ b/fragment.go @@ -478,7 +478,7 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } -func (f *Fragment) column(rowID, columnID uint64) (bool, error) { +func (f *Fragment) bit(rowID, columnID uint64) (bool, error) { pos, err := f.pos(rowID, columnID) if err != nil { return false, err @@ -486,22 +486,22 @@ func (f *Fragment) column(rowID, columnID uint64) (bool, error) { return f.storage.Contains(pos), nil } -// FieldValue uses a column of columns to read a multi-column value. -func (f *Fragment) FieldValue(columnID uint64, columnDepth uint) (value uint64, exists bool, err error) { +// FieldValue uses a column of bits to read a multi-bit value. +func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { f.mu.Lock() defer f.mu.Unlock() - // If existence column is unset then ignore remaining columns. - if v, err := f.column(uint64(columnDepth), columnID); err != nil { - return 0, false, errors.Wrap(err, "getting existence column") + // If existence bit is unset then ignore remaining bits. + if v, err := f.bit(uint64(bitDepth), columnID); err != nil { + return 0, false, errors.Wrap(err, "getting existence bit") } else if !v { return 0, false, nil } - // Compute other columns into a value. - for i := uint(0); i < columnDepth; i++ { - if v, err := f.column(uint64(i), columnID); err != nil { - return 0, false, errors.Wrapf(err, "getting value column %d", i) + // Compute other bits into a value. + for i := uint(0); i < bitDepth; i++ { + if v, err := f.bit(uint64(i), columnID); err != nil { + return 0, false, errors.Wrapf(err, "getting value bit %d", i) } else if v { value |= (1 << i) } @@ -510,12 +510,12 @@ func (f *Fragment) FieldValue(columnID uint64, columnDepth uint) (value uint64, return value, true, nil } -// SetFieldValue uses a column of columns to set a multi-column value. -func (f *Fragment) SetFieldValue(columnID uint64, columnDepth uint, value uint64) (changed bool, err error) { +// SetFieldValue uses a column of bits to set a multi-bit value. +func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - for i := uint(0); i < columnDepth; i++ { + for i := uint(0); i < bitDepth; i++ { if value&(1< uint(0); i-- { - ii := i - 1 // allow for uint range: (columndepth-1) to 0 + for i := bitDepth; i > uint(0); i-- { + ii := i - 1 // allow for uint range: (bitDepth-1) to 0 row := f.Row(uint64(ii)) x := consider.Difference(row) @@ -647,21 +647,21 @@ func (f *Fragment) FieldMin(filter *Row, columnDepth uint) (min, count uint64, e return min, count, nil } -// FieldMax returns the max of a given field as well as the number of columns involved. -// A bitmap can be passed in to optionally filter the computed columns. -func (f *Fragment) FieldMax(filter *Row, columnDepth uint) (max, count uint64, err error) { +// FieldMax returns the max of a given field as well as the number of bits involved. +// A bitmap can be passed in to optionally filter the computed bits. +func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { - consider := f.Row(uint64(columnDepth)) + consider := f.Row(uint64(bitDepth)) if filter != nil { consider = consider.Intersect(filter) } - // If there are no columns to consider, return early. + // If there are no bits to consider, return early. if consider.Count() == 0 { return 0, 0, nil } - for i := columnDepth; i > uint(0); i-- { + for i := bitDepth; i > uint(0); i-- { ii := i - 1 // allow for uint range: (columndepth-1) to 0 row := f.Row(uint64(ii)) @@ -679,31 +679,31 @@ func (f *Fragment) FieldMax(filter *Row, columnDepth uint) (max, count uint64, e } // FieldRange returns bitmaps with a field value encoding matching the predicate. -func (f *Fragment) FieldRange(op pql.Token, columnDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { switch op { case pql.EQ: - return f.fieldRangeEQ(columnDepth, predicate) + return f.fieldRangeEQ(bitDepth, predicate) case pql.NEQ: - return f.fieldRangeNEQ(columnDepth, predicate) + return f.fieldRangeNEQ(bitDepth, predicate) case pql.LT, pql.LTE: - return f.fieldRangeLT(columnDepth, predicate, op == pql.LTE) + return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE) case pql.GT, pql.GTE: - return f.fieldRangeGT(columnDepth, predicate, op == pql.GTE) + return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE) default: return nil, ErrInvalidRangeOperation } } -func (f *Fragment) fieldRangeEQ(columnDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(columnDepth)) + b := f.Row(uint64(bitDepth)) // Filter any columns that don't match the current column value. - for i := int(columnDepth - 1); i >= 0; i-- { + for i := int(bitDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) - column := (predicate >> uint(i)) & 1 + bit := (predicate >> uint(i)) & 1 - if column == 1 { + if bit == 1 { b = b.Intersect(row) } else { b = b.Difference(row) @@ -713,12 +713,12 @@ func (f *Fragment) fieldRangeEQ(columnDepth uint, predicate uint64) (*Row, error return b, nil } -func (f *Fragment) fieldRangeNEQ(columnDepth uint, predicate uint64) (*Row, error) { +func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. - b := f.Row(uint64(columnDepth)) + b := f.Row(uint64(bitDepth)) // Get the equal bitmap. - eq, err := f.fieldRangeEQ(columnDepth, predicate) + eq, err := f.fieldRangeEQ(bitDepth, predicate) if err != nil { return nil, err } @@ -729,21 +729,21 @@ func (f *Fragment) fieldRangeNEQ(columnDepth uint, predicate uint64) (*Row, erro return b, nil } -func (f *Fragment) fieldRangeLT(columnDepth uint, predicate uint64, allowEquality bool) (*Row, error) { +func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { keep := NewRow() // Start with set of columns with values set. - b := f.Row(uint64(columnDepth)) + b := f.Row(uint64(bitDepth)) // Filter any columns that don't match the current column value. leadingZeros := true - for i := int(columnDepth - 1); i >= 0; i-- { + for i := int(bitDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) - column := (predicate >> uint(i)) & 1 + bit := (predicate >> uint(i)) & 1 // Remove any columns with higher columns set. if leadingZeros { - if column == 0 { + if bit == 0 { b = b.Difference(row) continue } else { @@ -755,19 +755,19 @@ func (f *Fragment) fieldRangeLT(columnDepth uint, predicate uint64, allowEqualit // If column is zero then return only already kept columns. // If column is one then remove any one columns. if i == 0 && !allowEquality { - if column == 0 { + if bit == 0 { return keep, nil } return b.Difference(row.Difference(keep)), nil } - // If column is zero then remove all set columns not in excluded bitmap. - if column == 0 { + // If bit is zero then remove all set columns not in excluded bitmap. + if bit == 0 { b = b.Difference(row.Difference(keep)) continue } - // If column is set then add columns for set columns to exclude. + // If bit is set then add bits for set bits to exclude. // Don't bother to compute this on the final iteration. if i > 0 { keep = keep.Union(b.Difference(row)) @@ -777,32 +777,32 @@ func (f *Fragment) fieldRangeLT(columnDepth uint, predicate uint64, allowEqualit return b, nil } -func (f *Fragment) fieldRangeGT(columnDepth uint, predicate uint64, allowEquality bool) (*Row, error) { - b := f.Row(uint64(columnDepth)) +func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) { + b := f.Row(uint64(bitDepth)) keep := NewRow() // Filter any columns that don't match the current column value. - for i := int(columnDepth - 1); i >= 0; i-- { + for i := int(bitDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) - column := (predicate >> uint(i)) & 1 + bit := (predicate >> uint(i)) & 1 - // Handle last column differently. - // If column is one then return only already kept columns. - // If column is zero then remove any unset columns. + // Handle last bit differently. + // If bit is one then return only already kept bits. + // If bit is zero then remove any unset bits. if i == 0 && !allowEquality { - if column == 1 { + if bit == 1 { return keep, nil } return b.Difference(b.Difference(row).Difference(keep)), nil } - // If column is set then remove all unset columns not already kept. - if column == 1 { + // If bit is set then remove all unset bits not already kept. + if bit == 1 { b = b.Difference(b.Difference(row).Difference(keep)) continue } - // If column is unset then add columns with set column to keep. + // If bit is unset then add bits with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { keep = keep.Union(b.Intersect(row)) @@ -812,29 +812,29 @@ func (f *Fragment) fieldRangeGT(columnDepth uint, predicate uint64, allowEqualit return b, nil } -// FieldNotNull returns the not-null row (stored at columnDepth). -func (f *Fragment) FieldNotNull(columnDepth uint) (*Row, error) { - return f.Row(uint64(columnDepth)), nil +// FieldNotNull returns the not-null row (stored at bitDepth). +func (f *Fragment) FieldNotNull(bitDepth uint) (*Row, error) { + return f.Row(uint64(bitDepth)), nil } // FieldRangeBetween returns bitmaps with a field value encoding matching any value between predicateMin and predicateMax. -func (f *Fragment) FieldRangeBetween(columnDepth uint, predicateMin, predicateMax uint64) (*Row, error) { - b := f.Row(uint64(columnDepth)) +func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { + b := f.Row(uint64(bitDepth)) keep1 := NewRow() // GTE keep2 := NewRow() // LTE // Filter any columns that don't match the current column value. - for i := int(columnDepth - 1); i >= 0; i-- { + for i := int(bitDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) - column1 := (predicateMin >> uint(i)) & 1 - column2 := (predicateMax >> uint(i)) & 1 + bit1 := (predicateMin >> uint(i)) & 1 + bit2 := (predicateMax >> uint(i)) & 1 // GTE predicateMin - // If column is set then remove all unset columns not already kept. - if column1 == 1 { + // If bit is set then remove all unset bits not already kept. + if bit1 == 1 { b = b.Difference(b.Difference(row).Difference(keep1)) } else { - // If column is unset then add columns with set column to keep. + // If bit is unset then add bits with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { keep1 = keep1.Union(b.Intersect(row)) @@ -842,11 +842,11 @@ func (f *Fragment) FieldRangeBetween(columnDepth uint, predicateMin, predicateMa } // LTE predicateMin - // If column is zero then remove all set columns not in excluded bitmap. - if column2 == 0 { + // If bit is zero then remove all set bits not in excluded bitmap. + if bit2 == 0 { b = b.Difference(row.Difference(keep2)) } else { - // If column is set then add columns for set columns to exclude. + // If bit is set then add bits for set bits to exclude. // Don't bother to compute this on the final iteration. if i > 0 { keep2 = keep2.Union(b.Difference(row)) @@ -867,7 +867,7 @@ func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { return Pos(rowID, columnID), nil } -// ForEachBit executes fn for every column set in the fragment. +// ForEachBit executes fn for every bit set in the fragment. // Errors returned from fn are passed through. func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { f.mu.Lock() diff --git a/fragment_test.go b/fragment_test.go index 02042e6f5..abba4060a 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -38,12 +38,12 @@ var ( // SliceWidth is a helper reference to use when testing. const SliceWidth = pilosa.SliceWidth -// Ensure a fragment can set a column and retrieve it. +// Ensure a fragment can set a bit and retrieve it. func TestFragment_SetBit(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set columns on the fragment. + // Set bits on the fragment. if _, err := f.SetBit(120, 1); err != nil { t.Fatal(err) } else if _, err := f.SetBit(120, 6); err != nil { @@ -69,12 +69,12 @@ func TestFragment_SetBit(t *testing.T) { } } -// Ensure a fragment can clear a set column. +// Ensure a fragment can clear a set bits. func TestFragment_ClearBit(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set and then clear columns on the fragment. + // Set and then clear bits on the fragment. if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) } else if _, err := f.SetBit(1000, 2); err != nil { @@ -137,7 +137,7 @@ func TestFragment_SetFieldValue(t *testing.T) { t.Fatal("expected change") } - // Overwriting value should overwrite all columns. + // Overwriting value should overwrite all bits. if changed, err := f.SetFieldValue(100, 16, 2028); err != nil { t.Fatal(err) } else if !changed { @@ -176,13 +176,13 @@ func TestFragment_SetFieldValue(t *testing.T) { }) t.Run("QuickCheck", func(t *testing.T) { - if err := quick.Check(func(columnDepth uint, columnN uint64, values []uint64) bool { - // Limit column depth & maximum values. - columnDepth = (columnDepth % 62) + 1 - columnN = (columnN % 99) + 1 + if err := quick.Check(func(bitDepth uint, bitN uint64, values []uint64) bool { + // Limit bit depth & maximum values. + bitDepth = (bitDepth % 62) + 1 + bitN = (bitN % 99) + 1 for i := range values { - values[i] = values[i] % (1 << columnDepth) + values[i] = values[i] % (1 << bitDepth) } f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") @@ -191,24 +191,24 @@ func TestFragment_SetFieldValue(t *testing.T) { // Set values. m := make(map[uint64]int64) for _, value := range values { - columnID := value % columnN + bit_index := value % bitN - m[columnID] = int64(value) + m[bit_index] = int64(value) - if _, err := f.SetFieldValue(columnID, columnDepth, value); err != nil { + if _, err := f.SetFieldValue(bit_index, bitDepth, value); err != nil { t.Fatal(err) } } // Ensure values are set. - for columnID, value := range m { - v, exists, err := f.FieldValue(columnID, columnDepth) + for bit_index, value := range m { + v, exists, err := f.FieldValue(bit_index, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { - t.Fatalf("value mismatch: column=%d, columndepth=%d, value: %d != %d", columnID, columnDepth, value, v) + t.Fatalf("value mismatch: bit_index=%d, bitdepth=%d, value: %d != %d", bit_index, bitDepth, value, v) } else if !exists { - t.Fatalf("value should exist: column=%d", columnID) + t.Fatalf("value should exist: bit_index=%d", bit_index) } } @@ -221,24 +221,24 @@ func TestFragment_SetFieldValue(t *testing.T) { // Ensure a fragment can sum field values. func TestFragment_FieldSum(t *testing.T) { - const columnDepth = 16 + const bitDepth = 16 f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } t.Run("NoFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(nil, columnDepth); err != nil { + if sum, n, err := f.FieldSum(nil, bitDepth); err != nil { t.Fatal(err) } else if n != 4 { t.Fatalf("unexpected count: %d", n) @@ -248,7 +248,7 @@ func TestFragment_FieldSum(t *testing.T) { }) t.Run("WithFilter", func(t *testing.T) { - if sum, n, err := f.FieldSum(pilosa.NewRow(2000, 4000, 5000), columnDepth); err != nil { + if sum, n, err := f.FieldSum(pilosa.NewRow(2000, 4000, 5000), bitDepth); err != nil { t.Fatal(err) } else if n != 2 { t.Fatalf("unexpected count: %d", n) @@ -260,25 +260,25 @@ func TestFragment_FieldSum(t *testing.T) { // Ensure a fragment can find the min and max of field values. func TestFragment_FieldMinMax(t *testing.T) { - const columnDepth = 16 + const bitDepth = 16 f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, columnDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, columnDepth, 2817); err != nil { + } else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(7000, columnDepth, 0); err != nil { + } else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil { t.Fatal(err) } @@ -296,7 +296,7 @@ func TestFragment_FieldMinMax(t *testing.T) { {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if min, cnt, err := f.FieldMin(test.filter, columnDepth); err != nil { + if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil { t.Fatal(err) } else if min != test.exp { t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min) @@ -320,7 +320,7 @@ func TestFragment_FieldMinMax(t *testing.T) { {filter: pilosa.NewRow(7000), exp: 0, cnt: 1}, } for i, test := range tests { - if max, cnt, err := f.FieldMax(test.filter, columnDepth); err != nil { + if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil { t.Fatal(err) } else if max != test.exp { t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max) @@ -333,25 +333,25 @@ func TestFragment_FieldMinMax(t *testing.T) { // Ensure a fragment query for matching fields. func TestFragment_FieldRange(t *testing.T) { - const columnDepth = 16 + const bitDepth = 16 t.Run("EQ", func(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for equality. - if b, err := f.FieldRange(pql.EQ, columnDepth, 300); err != nil { + if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -363,18 +363,18 @@ func TestFragment_FieldRange(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil { + } else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil { t.Fatal(err) } // Query for inequality. - if b, err := f.FieldRange(pql.NEQ, columnDepth, 300); err != nil { + if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -386,43 +386,43 @@ func TestFragment_FieldRange(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, columnDepth, 2817); err != nil { + } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, columnDepth, 301); err != nil { + } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, columnDepth, 1); err != nil { + } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, columnDepth, 0); err != nil { + } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for fields less than (ending with set column). - if b, err := f.FieldRange(pql.LT, columnDepth, 301); err != nil { + if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields less than (ending with unset column). - if b, err := f.FieldRange(pql.LT, columnDepth, 300); err != nil { + if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields less than or equal to (ending with set column). - if b, err := f.FieldRange(pql.LTE, columnDepth, 301); err != nil { + if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields less than or equal to (ending with unset column). - if b, err := f.FieldRange(pql.LTE, columnDepth, 300); err != nil { + if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -434,43 +434,43 @@ func TestFragment_FieldRange(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, columnDepth, 2817); err != nil { + } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, columnDepth, 301); err != nil { + } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, columnDepth, 1); err != nil { + } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, columnDepth, 0); err != nil { + } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for fields greater than (ending with unset column). - if b, err := f.FieldRange(pql.GT, columnDepth, 300); err != nil { + if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than (ending with set column). - if b, err := f.FieldRange(pql.GT, columnDepth, 301); err != nil { + if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with unset column). - if b, err := f.FieldRange(pql.GTE, columnDepth, 300); err != nil { + if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with set column). - if b, err := f.FieldRange(pql.GTE, columnDepth, 301); err != nil { + if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -482,43 +482,43 @@ func TestFragment_FieldRange(t *testing.T) { defer f.Close() // Set values. - if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil { + if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil { + } else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(3000, columnDepth, 2817); err != nil { + } else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(4000, columnDepth, 301); err != nil { + } else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(5000, columnDepth, 1); err != nil { + } else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil { t.Fatal(err) - } else if _, err := f.SetFieldValue(6000, columnDepth, 0); err != nil { + } else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil { t.Fatal(err) } // Query for fields greater than (ending with unset column). - if b, err := f.FieldRangeBetween(columnDepth, 300, 2817); err != nil { + if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than (ending with set column). - if b, err := f.FieldRangeBetween(columnDepth, 301, 2817); err != nil { + if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with unset column). - if b, err := f.FieldRangeBetween(columnDepth, 301, 2816); err != nil { + if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with set column). - if b, err := f.FieldRangeBetween(columnDepth, 300, 2816); err != nil { + if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) @@ -589,9 +589,9 @@ func TestFragment_Top(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() // Set columns on the rows 100, 101, & 102. - f.MustSetColumns(100, 1, 3, 200) - f.MustSetColumns(101, 1) - f.MustSetColumns(102, 1, 2) + f.MustSetBits(100, 1, 3, 200) + f.MustSetBits(101, 1) + f.MustSetBits(102, 1, 2) f.RecalculateCache() // Retrieve top rows. @@ -612,9 +612,9 @@ func TestFragment_Top_Filter(t *testing.T) { defer f.Close() // Set columns on the rows 100, 101, & 102. - f.MustSetColumns(100, 1, 3, 200) - f.MustSetColumns(101, 1) - f.MustSetColumns(102, 1, 2) + f.MustSetBits(100, 1, 3, 200) + f.MustSetBits(101, 1) + f.MustSetBits(102, 1, 2) f.RecalculateCache() // Assign attributes. f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) @@ -645,10 +645,10 @@ func TestFragment_TopN_Intersect(t *testing.T) { src := pilosa.NewRow(1, 2, 3) // Set columns on various rows. - f.MustSetColumns(100, 1, 10, 11, 12) // one intersection - f.MustSetColumns(101, 1, 2, 3, 4) // three intersections - f.MustSetColumns(102, 1, 2, 4, 5, 6) // two intersections - f.MustSetColumns(103, 1000, 1001, 1002) // no intersection + f.MustSetBits(100, 1, 10, 11, 12) // one intersection + f.MustSetBits(101, 1, 2, 3, 4) // three intersections + f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections + f.MustSetBits(103, 1000, 1001, 1002) // no intersection f.RecalculateCache() // Retrieve top rows. @@ -681,7 +681,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { // Set columns on rows 0 - 999. Higher rows have higher column counts. for i := uint64(0); i < 1000; i++ { for j := uint64(0); j < i; j++ { - f.MustSetColumns(i, j) + f.MustSetBits(i, j) } } f.RecalculateCache() @@ -711,9 +711,9 @@ func TestFragment_TopN_IDs(t *testing.T) { defer f.Close() // Set columns on various rows. - f.MustSetColumns(100, 1, 2, 3) - f.MustSetColumns(101, 4, 5, 6, 7) - f.MustSetColumns(102, 8, 9, 10, 11, 12) + f.MustSetBits(100, 1, 2, 3) + f.MustSetBits(101, 4, 5, 6, 7) + f.MustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { @@ -732,9 +732,9 @@ func TestFragment_TopN_NopCache(t *testing.T) { defer f.Close() // Set columns on various rows. - f.MustSetColumns(100, 1, 2, 3) - f.MustSetColumns(101, 4, 5, 6, 7) - f.MustSetColumns(102, 8, 9, 10, 11, 12) + f.MustSetBits(100, 1, 2, 3) + f.MustSetBits(101, 4, 5, 6, 7) + f.MustSetBits(102, 8, 9, 10, 11, 12) // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { @@ -784,12 +784,12 @@ func TestFragment_TopN_CacheSize(t *testing.T) { defer f.Close() // Set columns on various rows. - f.MustSetColumns(100, 1, 2, 3) - f.MustSetColumns(101, 4, 5, 6, 7) - f.MustSetColumns(102, 8, 9, 10, 11, 12) - f.MustSetColumns(103, 8, 9, 10, 11, 12, 13) - f.MustSetColumns(104, 8, 9, 10, 11, 12, 13, 14) - f.MustSetColumns(105, 10, 11) + f.MustSetBits(100, 1, 2, 3) + f.MustSetBits(101, 4, 5, 6, 7) + f.MustSetBits(102, 8, 9, 10, 11, 12) + f.MustSetBits(103, 8, 9, 10, 11, 12, 13) + f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14) + f.MustSetBits(105, 10, 11) f.RecalculateCache() @@ -1084,9 +1084,9 @@ func TestFragment_Tanimoto(t *testing.T) { src := pilosa.NewRow(1, 2, 3) // Set columns on the rows 100, 101, & 102. - f.MustSetColumns(100, 1, 3, 2, 200) - f.MustSetColumns(101, 1, 3) - f.MustSetColumns(102, 1, 2, 10, 12) + f.MustSetBits(100, 1, 3, 2, 200) + f.MustSetBits(101, 1, 3) + f.MustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 50, Src: src}); err != nil { @@ -1107,9 +1107,9 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { src := pilosa.NewRow(1, 2, 3) // Set columns on the rows 100, 101, & 102. - f.MustSetColumns(100, 1, 3, 2, 200) - f.MustSetColumns(101, 1, 3) - f.MustSetColumns(102, 1, 2, 10, 12) + f.MustSetBits(100, 1, 3, 2, 200) + f.MustSetBits(101, 1, 3) + f.MustSetBits(102, 1, 2, 10, 12) f.RecalculateCache() if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 0, Src: src}); err != nil { diff --git a/handler_test.go b/handler_test.go index 2952f28b0..94ff9b309 100644 --- a/handler_test.go +++ b/handler_test.go @@ -194,13 +194,13 @@ func TestHandler_MaxSlices(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetColumns(30, (1*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetColumns(30, (1*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetColumns(30, (3*SliceWidth)+4) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetColumns(40, (0*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetColumns(40, (0*SliceWidth)+2) - hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetColumns(40, (0*SliceWidth)+8) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2) + hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8) h := test.NewHandler() h.API.Holder = hldr.Holder @@ -1112,7 +1112,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { // Set columns in the index. f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - f0.MustSetColumns(100, 1, 2, 3) + f0.MustSetBits(100, 1, 2, 3) // Begin backing up from slice i/f/0. resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0") diff --git a/test/fragment.go b/test/fragment.go index ba5336af8..1018ef6c0 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -85,9 +85,9 @@ func (f *Fragment) Reopen() error { return nil } -// MustSetColumns sets columns on a row. Panic on error. +// MustSetBits sets columns on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (f *Fragment) MustSetColumns(rowID uint64, columnIDs ...uint64) { +func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { if _, err := f.SetBit(rowID, columnID); err != nil { panic(err) diff --git a/test/frame.go b/test/frame.go index 825fb06ae..e107b7d85 100644 --- a/test/frame.go +++ b/test/frame.go @@ -75,7 +75,7 @@ func (f *Frame) Reopen() error { return nil } -// MustSetBit sets a column on the frame. Panic on error. +// MustSetBit sets a bit on the frame. Panic on error. func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { changed, err := f.SetBit(view, rowID, columnID, t) if err != nil { diff --git a/time_test.go b/time_test.go index 87354f779..233a2ae1a 100644 --- a/time_test.go +++ b/time_test.go @@ -65,7 +65,7 @@ func TestViewByTimeUnit(t *testing.T) { }) } -// Ensure all applicable frame names can be generated when mutating a time column. +// Ensure all applicable frame names can be generated when mutating a time bit. func TestViewsByTime(t *testing.T) { ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC) diff --git a/view_test.go b/view_test.go index 7c9e901ae..f1c875c13 100644 --- a/view_test.go +++ b/view_test.go @@ -72,9 +72,9 @@ func (v *View) Reopen() error { return v.Open() } -// MustSetColumns sets columns on a row. Panic on error. +// MustSetBits sets columns on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (v *View) MustSetColumns(rowID uint64, columnIDs ...uint64) { +func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { if _, err := v.SetBit(rowID, columnID); err != nil { panic(err) From 4ef266e5ccc0e963b48f4b577bfb0c4d8fbcba91 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Thu, 24 May 2018 16:35:27 -0500 Subject: [PATCH 05/13] revert a bunch of stuff and fix some comments --- cache.go | 2 +- client.go | 29 +++++++++--------- client_test.go | 4 +-- cluster.go | 2 +- cluster_test.go | 2 +- cmd/bench.go | 2 +- cmd/bench_test.go | 4 +-- cmd/import.go | 4 +-- ctl/bench.go | 2 +- ctl/bench_test.go | 2 +- ctl/import.go | 75 +++++++++++++++++++++++------------------------ executor.go | 6 ++-- executor_test.go | 8 ++--- fragment.go | 74 +++++++++++++++++++++++----------------------- fragment_test.go | 58 ++++++++++++++++++------------------ frame.go | 24 +++++++-------- handler_test.go | 2 +- stats_test.go | 4 +-- view.go | 36 +++++++++++------------ view_test.go | 2 +- 20 files changed, 171 insertions(+), 171 deletions(-) diff --git a/cache.go b/cache.go index 1d45c04be..06046220a 100644 --- a/cache.go +++ b/cache.go @@ -469,7 +469,7 @@ type BitmapCache interface { // SimpleCache implements BitmapCache // it is meant to be a short-lived cache for cases where writes are continuing to access -// the same column within a short time frame (i.e. good for write-heavy loads) +// the same row within a short time frame (i.e. good for write-heavy loads) // A read-heavy use case would cause the cache to get bigger, potentially causing the // node to run out of memory. type SimpleCache struct { diff --git a/client.go b/client.go index c385c5177..a18d066f1 100644 --- a/client.go +++ b/client.go @@ -308,7 +308,7 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl return nil } -// ImportK bulk imports columns to a host. +// ImportK bulk imports bits specified by string keys to a host. func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error { if index == "" { return ErrIndexRequired @@ -356,7 +356,7 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte columnIDs := Bits(bits).ColumnIDs() timestamps := Bits(bits).Timestamps() - // Marshal columns to protobufs. + // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, Frame: frame, @@ -378,7 +378,7 @@ func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) { columnKeys := Bits(bits).ColumnKeys() timestamps := Bits(bits).Timestamps() - // Marshal columns to protobufs. + // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, Frame: frame, @@ -465,7 +465,7 @@ func marshalImportValuePayload(index, frame, field string, slice uint64, vals [] columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() - // Marshal columns to protobufs. + // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportValueRequest{ Index: index, Frame: frame, @@ -1140,7 +1140,8 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto return nil } -// Bit represents the location of a the intersection of a row and a column. +// Bit represents the intersection of a row and a column. It can be specifed by +// integer ids or string keys. type Bit struct { RowID uint64 ColumnID uint64 @@ -1149,7 +1150,7 @@ type Bit struct { Timestamp int64 } -// Bits represents a slice of Bit. +// Bits is a slice of Bit. type Bits []Bit func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -1210,17 +1211,17 @@ func (p Bits) Timestamps() []int64 { return other } -// GroupBySlice returns a map of columns by slice. +// GroupBySlice returns a map of bits by slice. func (p Bits) GroupBySlice() map[uint64][]Bit { m := make(map[uint64][]Bit) - for _, column := range p { - slice := column.ColumnID / SliceWidth - m[slice] = append(m[slice], column) + for _, bit := range p { + slice := bit.ColumnID / SliceWidth + m[slice] = append(m[slice], bit) } - for slice, columns := range m { - sort.Sort(Bits(columns)) - m[slice] = columns + for slice, bits := range m { + sort.Sort(Bits(bits)) + m[slice] = bits } return m @@ -1277,7 +1278,7 @@ func (p FieldValues) GroupBySlice() map[uint64][]FieldValue { return m } -// BitsByPos represents a slice of columns sorted by internal position. +// BitsByPos is a slice of bits sorted row then column. type BitsByPos []Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/client_test.go b/client_test.go index 82156d138..960d93421 100644 --- a/client_test.go +++ b/client_test.go @@ -502,11 +502,11 @@ func TestClient_FragmentBlocks(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set two columns on blocks 0 & 3. + // Set two bits on blocks 0 & 3. hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100) - // Set a column on a different slice. + // Set a bit on a different slice. hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1) s := test.NewServer() diff --git a/cluster.go b/cluster.go index 049136acc..97afa0eef 100644 --- a/cluster.go +++ b/cluster.go @@ -930,7 +930,7 @@ func (c *Cluster) Open() error { // (and now in a state of STARTING) so that it can be put to the correct // cluster state. // TODO: Because the normal code path already sends a NodeJoin event (via - // memberlist), this it a column redundant in most cases. Perhaps determine + // memberlist), this it a bit redundant in most cases. Perhaps determine // that the node has been restarted and don't do this step. msg := &internal.NodeEventMessage{ Event: uint32(NodeJoin), diff --git a/cluster_test.go b/cluster_test.go index 21b46060b..b09820ac8 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -476,7 +476,7 @@ func TestCluster_ResizeStates(t *testing.T) { t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs) } - // Columns + // Bits // Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0. node1Frame := node1.Holder.Frame("i", "f") node1View := node1Frame.View("standard") diff --git a/cmd/bench.go b/cmd/bench.go index f905b49e6..d4b2b8580 100644 --- a/cmd/bench.go +++ b/cmd/bench.go @@ -45,7 +45,7 @@ Executes a benchmark for a given operation against the index. flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.") flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.") - flags.StringVarP(&Bencher.Op, "operation", "o", "set-column", "Operation to perform: choose from [set-column]") + flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]") flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.") ctl.SetTLSConfig(flags, &Bencher.TLS.CertificatePath, &Bencher.TLS.CertificateKeyPath, &Bencher.TLS.SkipVerify) diff --git a/cmd/bench_test.go b/cmd/bench_test.go index c59aae131..4b94d9392 100644 --- a/cmd/bench_test.go +++ b/cmd/bench_test.go @@ -33,7 +33,7 @@ func TestBenchHelp(t *testing.T) { func TestBenchConfig(t *testing.T) { tests := []commandTest{ { - args: []string{"bench", "--operation", "set-column"}, + args: []string{"bench", "--operation", "set-bit"}, env: map[string]string{"PILOSA_HOST": "localhost:12345"}, cfgFileContent: ` index = "myindex" @@ -44,7 +44,7 @@ frame = "f1" v.Check(cmd.Bencher.Host, "localhost:12345") v.Check(cmd.Bencher.Index, "myindex") v.Check(cmd.Bencher.Frame, "f1") - v.Check(cmd.Bencher.Op, "set-column") + v.Check(cmd.Bencher.Op, "set-bit") v.Check(cmd.Bencher.N, 0) return v.Error() }, diff --git a/cmd/import.go b/cmd/import.go index fdf3e1e92..c6cad303a 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command importCmd := &cobra.Command{ Use: "import", Short: "Bulk load data into pilosa.", - Long: `Bulk imports one or more CSV files to a host's index and frame. The columns + Long: `Bulk imports one or more CSV files to a host's index and frame. The data of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: @@ -57,7 +57,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.") flags.StringVarP(&Importer.Field, "field", "", "", "Field to import into.") flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.") - flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of columns to buffer/sort before importing.") + flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.") flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") diff --git a/ctl/bench.go b/ctl/bench.go index 50a18396a..4a71169d9 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -62,7 +62,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error { } switch cmd.Op { - case "set-column": + case "set-bit": return cmd.runSetBit(ctx, client) case "": return errors.New("op required") diff --git a/ctl/bench_test.go b/ctl/bench_test.go index ac7fc3ad1..4790ccb44 100644 --- a/ctl/bench_test.go +++ b/ctl/bench_test.go @@ -56,7 +56,7 @@ func TestBenchCommand_Run(t *testing.T) { r, w, _ := os.Pipe() cm := NewBenchCommand(stdin, w, w) - cm.Op = "set-column" + cm.Op = "set-bit" cm.Host = "localhost:10101" err := cm.Run(context.Background()) diff --git a/ctl/import.go b/ctl/import.go index e193cd070..a9d70fd7c 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -107,7 +107,6 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { // Import each path and import by slice. for _, path := range cmd.Paths { - // Parse path into columns. logger.Printf("parsing: %s", path) if err := cmd.importPath(ctx, path); err != nil { return err @@ -129,22 +128,22 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error { return nil } -// importPath parses a path into columns and imports it to the server. +// importPath parses a path into bits and imports it to the server. func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { // If a field is provided, treat the import data as values to be range-encoded. if cmd.Field != "" { return cmd.bufferFieldValues(ctx, path) } else { if cmd.StringKeys { - return cmd.bufferColumnsK(ctx, path) + return cmd.bufferBitsK(ctx, path) } else { - return cmd.bufferColumns(ctx, path) + return cmd.bufferBits(ctx, path) } } } -// bufferColumns buffers slices of columns to be imported as a batch. -func (cmd *ImportCommand) bufferColumns(ctx context.Context, path string) error { +// bufferBits buffers slices of bits to be imported as a batch. +func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error { a := make([]pilosa.Bit, 0, cmd.BufferSize) var r *csv.Reader @@ -157,7 +156,7 @@ func (cmd *ImportCommand) bufferColumns(ctx context.Context, path string) error } defer f.Close() - // Read rows as columns. + // Read rows as bits. r = csv.NewReader(f) } else { r = csv.NewReader(cmd.Stdin) @@ -183,21 +182,21 @@ func (cmd *ImportCommand) bufferColumns(ctx context.Context, path string) error return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record)) } - var column pilosa.Bit + var bit pilosa.Bit // Parse row id. rowID, err := strconv.ParseUint(record[0], 10, 64) if err != nil { return fmt.Errorf("invalid row id on row %d: %q", rnum, record[0]) } - column.RowID = rowID + bit.RowID = rowID // Parse column id. columnID, err := strconv.ParseUint(record[1], 10, 64) if err != nil { return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1]) } - column.ColumnID = columnID + bit.ColumnID = columnID // Parse time, if exists. if len(record) > 2 && record[2] != "" { @@ -205,37 +204,37 @@ func (cmd *ImportCommand) bufferColumns(ctx context.Context, path string) error if err != nil { return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2]) } - column.Timestamp = t.UnixNano() + bit.Timestamp = t.UnixNano() } - a = append(a, column) + a = append(a, bit) - // If we've reached the buffer size then import columns. + // If we've reached the buffer size then import bits. if len(a) == cmd.BufferSize { - if err := cmd.importColumns(ctx, a); err != nil { + if err := cmd.importBits(ctx, a); err != nil { return err } a = a[:0] } } - // If there are still columns in the buffer then flush them. - if err := cmd.importColumns(ctx, a); err != nil { + // If there are still bits in the buffer then flush them. + if err := cmd.importBits(ctx, a); err != nil { return err } return nil } -// importColumns sends batches of columns to the server. -func (cmd *ImportCommand) importColumns(ctx context.Context, bits []pilosa.Bit) error { +// importBits sends batches of bits to the server. +func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) - // Group columns by slice. - logger.Printf("grouping %d columns", len(bits)) + // Group bits by slice. + logger.Printf("grouping %d bits", len(bits)) bitsBySlice := pilosa.Bits(bits).GroupBySlice() - // Parse path into columns. + // Parse path into bits. for slice, chunk := range bitsBySlice { if cmd.Sort { sort.Sort(pilosa.BitsByPos(chunk)) @@ -250,8 +249,8 @@ func (cmd *ImportCommand) importColumns(ctx context.Context, bits []pilosa.Bit) return nil } -// bufferColumnsK buffers slices of keys to be imported as a batch. -func (cmd *ImportCommand) bufferColumnsK(ctx context.Context, path string) error { +// bufferBitsK buffers slices of keys to be imported as a batch. +func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error { a := make([]pilosa.Bit, 0, cmd.BufferSize) var r *csv.Reader @@ -264,7 +263,7 @@ func (cmd *ImportCommand) bufferColumnsK(ctx context.Context, path string) error } defer f.Close() - // Read rows as columns. + // Read rows as bits. r = csv.NewReader(f) } else { r = csv.NewReader(cmd.Stdin) @@ -290,19 +289,19 @@ func (cmd *ImportCommand) bufferColumnsK(ctx context.Context, path string) error return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record)) } - var column pilosa.Bit + var bit pilosa.Bit // Parse row key. if record[0] == "" { return fmt.Errorf("invalid row key on row %d: %q", rnum, record[0]) } - column.RowKey = record[0] + bit.RowKey = record[0] // Parse column key. if record[1] == "" { return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1]) } - column.ColumnKey = record[1] + bit.ColumnKey = record[1] // Parse time, if exists. if len(record) > 2 && record[2] != "" { @@ -310,36 +309,36 @@ func (cmd *ImportCommand) bufferColumnsK(ctx context.Context, path string) error if err != nil { return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2]) } - column.Timestamp = t.UnixNano() + bit.Timestamp = t.UnixNano() } - a = append(a, column) + a = append(a, bit) - // If we've reached the buffer size then import columns. + // If we've reached the buffer size then import bits. if len(a) == cmd.BufferSize { - if err := cmd.importColumnsK(ctx, a); err != nil { + if err := cmd.importBitsK(ctx, a); err != nil { return err } a = a[:0] } } - // If there are still columnKs in the buffer then flush them. - if err := cmd.importColumnsK(ctx, a); err != nil { + // If there are still bitKs in the buffer then flush them. + if err := cmd.importBitsK(ctx, a); err != nil { return err } return nil } -// importColumnsK sends batches of columnKs to the server. -func (cmd *ImportCommand) importColumnsK(ctx context.Context, columns []pilosa.Bit) error { +// importBitsK sends batches of bitKs to the server. +func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) error { logger := log.New(cmd.Stderr, "", log.LstdFlags) // TODO: does it help to sort the rowKeys? - logger.Printf("importing keys: n=%d", len(columns)) - if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, columns); err != nil { + logger.Printf("importing keys: n=%d", len(bits)) + if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil { return errors.Wrap(err, "importing keys") } @@ -360,7 +359,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er } defer f.Close() - // Read rows as columns. + // Read rows as bits. r = csv.NewReader(f) } else { r = csv.NewReader(cmd.Stdin) diff --git a/executor.go b/executor.go index bdc7618fe..8a7c35d21 100644 --- a/executor.go +++ b/executor.go @@ -1064,7 +1064,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) } - // Clear columns for each view. + // Clear bits for each view. switch view { case ViewStandard: return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt) @@ -1164,7 +1164,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, timestamp = &t } - // Set columns for each view. + // Set bits for each view. switch view { case ViewStandard: return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt) @@ -1404,7 +1404,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil { return nil, err } - frame.Stats.Count("SetColumnAttrs", 1, 1.0) + frame.Stats.Count("SetRowAttrs", 1, 1.0) } // Do not forward call if this is already being forwarded. diff --git a/executor_test.go b/executor_test.go index 3225cc747..f2975548a 100644 --- a/executor_test.go +++ b/executor_test.go @@ -40,7 +40,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set columns. + // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ @@ -60,7 +60,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } - // Inhicolumn columns. + // Inhibit columns attributes. if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { @@ -69,7 +69,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } - // Inhicolumn attributes. + // Inhibit row attributes. if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { @@ -89,7 +89,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set columns. + // Set bits. if _, err := e.Execute(context.Background(), "i", test.MustParse(``+ fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+ fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+ diff --git a/fragment.go b/fragment.go index 8651f0a4a..9aa5e17e7 100644 --- a/fragment.go +++ b/fragment.go @@ -171,7 +171,7 @@ func (f *Fragment) Open() error { // Clear checksums. f.checksums = make(map[int][]byte) - // Read last column to determine max row. + // Read last bit to determine max row. pos := f.storage.Max() f.maxRowID = pos / SliceWidth f.stats.Gauge("rows", float64(f.maxRowID), 1.0) @@ -380,7 +380,7 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R return row } -// SetBit sets a column for a given column & row within the fragment. +// SetBit sets a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() @@ -390,10 +390,10 @@ func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { changed = false - // Determine the position of the column in the storage. + // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) if err != nil { - return false, errors.Wrap(err, "getting column ops") + return false, errors.Wrap(err, "getting bit pos") } // Write to storage. @@ -432,7 +432,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { return changed, nil } -// ClearBit clears a column for a given column & row within the fragment. +// ClearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { f.mu.Lock() @@ -442,10 +442,10 @@ func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { changed = false - // Determine the position of the column in the storage. + // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) if err != nil { - return false, errors.Wrap(err, "getting column pos") + return false, errors.Wrap(err, "getting bit pos") } // Write to storage. @@ -582,10 +582,10 @@ func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uin return changed, nil } -// FieldSum returns the sum of a given field as well as the number of bits involved. +// FieldSum returns the sum of a given field as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) { - // Compute count based on the existence column. + // Compute count based on the existence row. row := f.Row(uint64(bitDepth)) if filter != nil { count = row.IntersectionCount(filter) @@ -614,7 +614,7 @@ func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err return sum, count, nil } -// FieldMin returns the min of a given field as well as the number of bits involved. +// FieldMin returns the min of a given field as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) { @@ -647,8 +647,8 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err return min, count, nil } -// FieldMax returns the max of a given field as well as the number of bits involved. -// A bitmap can be passed in to optionally filter the computed bits. +// FieldMax returns the max of a given field as well as the number of columns involved. +// A bitmap can be passed in to optionally filter the computed columns. func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { consider := f.Row(uint64(bitDepth)) @@ -656,13 +656,13 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err consider = consider.Intersect(filter) } - // If there are no bits to consider, return early. + // If there are no columns to consider, return early. if consider.Count() == 0 { return 0, 0, nil } for i := bitDepth; i > uint(0); i-- { - ii := i - 1 // allow for uint range: (columndepth-1) to 0 + ii := i - 1 // allow for uint range: (bitDepth-1) to 0 row := f.Row(uint64(ii)) x := row.Intersect(consider) @@ -741,7 +741,7 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b row := f.Row(uint64(i)) bit := (predicate >> uint(i)) & 1 - // Remove any columns with higher columns set. + // Remove any columns with higher bits set. if leadingZeros { if bit == 0 { b = b.Difference(row) @@ -751,9 +751,9 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b } } - // Handle last column differently. - // If column is zero then return only already kept columns. - // If column is one then remove any one columns. + // Handle last bit differently. + // If bit is zero then return only already kept columns. + // If bit is one then remove any one columns. if i == 0 && !allowEquality { if bit == 0 { return keep, nil @@ -767,7 +767,7 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b continue } - // If bit is set then add bits for set bits to exclude. + // If bit is set then add columns for set bits to exclude. // Don't bother to compute this on the final iteration. if i > 0 { keep = keep.Union(b.Difference(row)) @@ -781,14 +781,14 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b b := f.Row(uint64(bitDepth)) keep := NewRow() - // Filter any columns that don't match the current column value. + // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) bit := (predicate >> uint(i)) & 1 // Handle last bit differently. - // If bit is one then return only already kept bits. - // If bit is zero then remove any unset bits. + // If bit is one then return only already kept columns. + // If bit is zero then remove any unset columns. if i == 0 && !allowEquality { if bit == 1 { return keep, nil @@ -796,13 +796,13 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b return b.Difference(b.Difference(row).Difference(keep)), nil } - // If bit is set then remove all unset bits not already kept. + // If bit is set then remove all unset columns not already kept. if bit == 1 { b = b.Difference(b.Difference(row).Difference(keep)) continue } - // If bit is unset then add bits with set bit to keep. + // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { keep = keep.Union(b.Intersect(row)) @@ -1184,7 +1184,7 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i } } -// BlockData returns columns in a block as row & column ID pairs. +// BlockData returns bits in a block as row & column ID pairs. func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() @@ -1196,11 +1196,11 @@ func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { return } -// MergeBlock compares the block's columns and computes a diff with another set of block columns. -// The state of a column is determined by consensus from all blocks being considered. +// MergeBlock compares the block's bits and computes a diff with another set of block bits. +// The state of a bit is determined by consensus from all blocks being considered. // -// For example, if 3 blocks are compared and two have a set column and one has a -// cleared column then the column is considered cleared. The function returns the +// For example, if 3 blocks are compared and two have a set bit and one has a +// cleared bit then the bit is considered cleared. The function returns the // diff per incoming block so that all can be in sync. func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) { // Ensure that all pair sets are of equal length. @@ -1305,14 +1305,14 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e } } - // Set local columns. + // Set local bits. for i := range sets[0].ColumnIDs { if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "setting") } } - // Clear local columns. + // Clear local bits. for i := range clears[0].ColumnIDs { if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil { return nil, nil, errors.Wrap(err, "clearing") @@ -1322,7 +1322,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e return sets[1:], clears[1:], nil } -// Import bulk imports a set of columns and then snapshots the storage. +// Import bulk imports a set of bits and then snapshots the storage. // This does not affect the fragment's cache. func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { f.mu.Lock() @@ -1335,7 +1335,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { // Disconnect op writer so we don't append updates. f.storage.OpWriter = nil - // Process every column. + // Process every bit. // If an error occurs then reopen the storage. lastID := uint64(0) if err := func() error { @@ -1343,10 +1343,10 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { for i := range rowIDs { rowID, columnID := rowIDs[i], columnIDs[i] - // Determine the position of the column in the storage. + // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) if err != nil { - return errors.Wrap(err, "getting column pos") + return errors.Wrap(err, "getting bit pos") } // Write to storage. @@ -1393,7 +1393,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { } // ImportValue bulk imports a set of range-encoded values. -func (f *Fragment) ImportValue(columnIDs, values []uint64, columnDepth uint) error { +func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error { f.mu.Lock() defer f.mu.Unlock() // Verify that there are an equal number of column ids and values. @@ -1408,7 +1408,7 @@ func (f *Fragment) ImportValue(columnIDs, values []uint64, columnDepth uint) err for i := range columnIDs { columnID, value := columnIDs[i], values[i] - _, err := f.importSetFieldValue(columnID, columnDepth, value) + _, err := f.importSetFieldValue(columnID, bitDepth, value) if err != nil { return errors.Wrap(err, "setting") } diff --git a/fragment_test.go b/fragment_test.go index abba4060a..6b6294d76 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -69,7 +69,7 @@ func TestFragment_SetBit(t *testing.T) { } } -// Ensure a fragment can clear a set bits. +// Ensure a fragment can clear a set bit. func TestFragment_ClearBit(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() @@ -191,24 +191,24 @@ func TestFragment_SetFieldValue(t *testing.T) { // Set values. m := make(map[uint64]int64) for _, value := range values { - bit_index := value % bitN + columnID := value % bitN - m[bit_index] = int64(value) + m[columnID] = int64(value) - if _, err := f.SetFieldValue(bit_index, bitDepth, value); err != nil { + if _, err := f.SetFieldValue(columnID, bitDepth, value); err != nil { t.Fatal(err) } } // Ensure values are set. - for bit_index, value := range m { - v, exists, err := f.FieldValue(bit_index, bitDepth) + for columnID, value := range m { + v, exists, err := f.FieldValue(columnID, bitDepth) if err != nil { t.Fatal(err) } else if value != int64(v) { - t.Fatalf("value mismatch: bit_index=%d, bitdepth=%d, value: %d != %d", bit_index, bitDepth, value, v) + t.Fatalf("value mismatch: columnID=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v) } else if !exists { - t.Fatalf("value should exist: bit_index=%d", bit_index) + t.Fatalf("value should exist: columnID=%d", columnID) } } @@ -531,7 +531,7 @@ func TestFragment_Snapshot(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set and then clear columns on the fragment. + // Set and then clear bits on the fragment. if _, err := f.SetBit(1000, 1); err != nil { t.Fatal(err) } else if _, err := f.SetBit(1000, 2); err != nil { @@ -555,12 +555,12 @@ func TestFragment_Snapshot(t *testing.T) { } } -// Ensure a fragment can iterate over all columns in order. +// Ensure a fragment can iterate over all bits in order. func TestFragment_ForEachBit(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set columns on the fragment. + // Set bits on the fragment. if _, err := f.SetBit(100, 20); err != nil { t.Fatal(err) } else if _, err := f.SetBit(2, 38); err != nil { @@ -569,7 +569,7 @@ func TestFragment_ForEachBit(t *testing.T) { t.Fatal(err) } - // Iterate over columns. + // Iterate over bits. var result [][2]uint64 if err := f.ForEachBit(func(rowID, columnID uint64) error { result = append(result, [2]uint64{rowID, columnID}) @@ -578,7 +578,7 @@ func TestFragment_ForEachBit(t *testing.T) { t.Fatal(err) } - // Verify columns are correct. + // Verify bits are correct. if !reflect.DeepEqual(result, [][2]uint64{{2, 37}, {2, 38}, {100, 20}}) { t.Fatalf("unexpected result: %#v", result) } @@ -588,7 +588,7 @@ func TestFragment_ForEachBit(t *testing.T) { func TestFragment_Top(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() - // Set columns on the rows 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 200) f.MustSetBits(101, 1) f.MustSetBits(102, 1, 2) @@ -611,7 +611,7 @@ func TestFragment_Top_Filter(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() - // Set columns on the rows 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 200) f.MustSetBits(101, 1) f.MustSetBits(102, 1, 2) @@ -644,7 +644,7 @@ func TestFragment_TopN_Intersect(t *testing.T) { // Create an intersecting input row. src := pilosa.NewRow(1, 2, 3) - // Set columns on various rows. + // Set bits on various rows. f.MustSetBits(100, 1, 10, 11, 12) // one intersection f.MustSetBits(101, 1, 2, 3, 4) // three intersections f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections @@ -678,7 +678,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, ) - // Set columns on rows 0 - 999. Higher rows have higher column counts. + // Set bits on rows 0 - 999. Higher rows have higher bit counts. for i := uint64(0); i < 1000; i++ { for j := uint64(0); j < i; j++ { f.MustSetBits(i, j) @@ -710,7 +710,7 @@ func TestFragment_TopN_IDs(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked) defer f.Close() - // Set columns on various rows. + // Set bits on various rows. f.MustSetBits(100, 1, 2, 3) f.MustSetBits(101, 4, 5, 6, 7) f.MustSetBits(102, 8, 9, 10, 11, 12) @@ -731,7 +731,7 @@ func TestFragment_TopN_NopCache(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone) defer f.Close() - // Set columns on various rows. + // Set bits on various rows. f.MustSetBits(100, 1, 2, 3) f.MustSetBits(101, 4, 5, 6, 7) f.MustSetBits(102, 8, 9, 10, 11, 12) @@ -783,7 +783,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { } defer f.Close() - // Set columns on various rows. + // Set bits on various rows. f.MustSetBits(100, 1, 2, 3) f.MustSetBits(101, 4, 5, 6, 7) f.MustSetBits(102, 8, 9, 10, 11, 12) @@ -816,7 +816,7 @@ func TestFragment_Checksum(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Retrieve checksum and set columns. + // Retrieve checksum and set bits. orig := f.Checksum() if _, err := f.SetBit(1, 200); err != nil { t.Fatal(err) @@ -848,7 +848,7 @@ func TestFragment_Blocks(t *testing.T) { } prev = blocks - // Set column on different row. + // Set bit on different row. if _, err := f.SetBit(20, 0); err != nil { t.Fatal(err) } @@ -858,7 +858,7 @@ func TestFragment_Blocks(t *testing.T) { } prev = blocks - // Set column on different column. + // Set bit on different column. if _, err := f.SetBit(20, 100); err != nil { t.Fatal(err) } @@ -873,7 +873,7 @@ func TestFragment_Blocks_Empty(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set columns on a different block. + // Set bits on a different block. if _, err := f.SetBit(100, 1); err != nil { t.Fatal(err) } @@ -891,7 +891,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU) defer f.Close() - // Set columns on the fragment. + // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) @@ -941,7 +941,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) { t.Fatal(err) } - // Set columns on the fragment. + // Set bits on the fragment. for i := uint64(0); i < 1000; i++ { if _, err := f.SetBit(i, 0); err != nil { t.Fatal(err) @@ -1083,7 +1083,7 @@ func TestFragment_Tanimoto(t *testing.T) { src := pilosa.NewRow(1, 2, 3) - // Set columns on the rows 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 2, 200) f.MustSetBits(101, 1, 3) f.MustSetBits(102, 1, 2, 10, 12) @@ -1106,7 +1106,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { src := pilosa.NewRow(1, 2, 3) - // Set columns on the rows 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 2, 200) f.MustSetBits(101, 1, 3) f.MustSetBits(102, 1, 2, 10, 12) @@ -1129,7 +1129,7 @@ func TestFragment_Snapshot_Run(t *testing.T) { f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f.Close() - // Set columns on the fragment. + // Set bits on the fragment. for i := uint64(1); i < 3; i++ { if _, err := f.SetBit(1000, i); err != nil { t.Fatal(err) diff --git a/frame.go b/frame.go index 4d0cc9e98..de9d3527f 100644 --- a/frame.go +++ b/frame.go @@ -587,7 +587,7 @@ func (f *Frame) DeleteView(name string) error { return nil } -// SetBit sets a column on a view within the frame. +// SetBit sets a bit on a view within the frame. func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. if !IsValidView(name) { @@ -600,7 +600,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed return changed, errors.Wrap(err, "creating view") } - // Set non-time column. + // Set non-time bit. if v, err := view.SetBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "setting on view") } else if v { @@ -612,7 +612,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed return changed, nil } - // If a timestamp is specified then set columns across all views for the quantum. + // If a timestamp is specified then set bits across all views for the quantum. for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { view, err := f.CreateViewIfNotExists(subname) if err != nil { @@ -629,7 +629,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed return changed, nil } -// ClearBit clears a column within the frame. +// ClearBit clears a bit within the frame. func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) { // Validate view name. if !IsValidView(name) { @@ -642,7 +642,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change return changed, errors.Wrap(err, "creating view") } - // Clear non-time column. + // Clear non-time bit. if v, err := view.ClearBit(rowID, colID); err != nil { return changed, errors.Wrap(err, "setting on view") } else if v { @@ -654,7 +654,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change return changed, nil } - // If a timestamp is specified then clear columns across all views for the quantum. + // If a timestamp is specified then clear bits across all views for the quantum. for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) { view, err := f.CreateViewIfNotExists(subname) if err != nil { @@ -846,13 +846,13 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro inverse = []string{ViewInverse} } else { standard = ViewsByTime(ViewStandard, *timestamp, q) - // In order to match the logic of `SetBit()`, we want columns + // In order to match the logic of `SetBit()`, we want bits // with timestamps to write to both time and standard views. standard = append(standard, ViewStandard) inverse = ViewsByTime(ViewInverse, *timestamp, q) } - // Attach column to each standard view. + // Attach bit to each standard view. for _, name := range standard { key := importKey{View: name, Slice: columnID / SliceWidth} data := dataByFragment[key] @@ -862,7 +862,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro } if f.inverseEnabled { - // Attach reversed columns to each inverse view. + // Attach reversed bits to each inverse view. for _, name := range inverse { key := importKey{View: name, Slice: rowID / SliceWidth} data := dataByFragment[key] @@ -909,7 +909,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro // ImportValue bulk imports range-encoded value data. func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error { viewName := ViewFieldPrefix + fieldName - // Get the field so we know columnDepth. + // Get the field so we know bitDepth. field := f.Field(fieldName) if field == nil { return fmt.Errorf("Field does not exist: %s", fieldName) @@ -939,7 +939,7 @@ func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64 for key, data := range dataByFragment { // The view must already exist (i.e. we can't create it) - // because we need to know columnDepth (based on min/max value). + // because we need to know bitDepth (based on min/max value). view, err := f.CreateViewIfNotExists(key.View) if err != nil { return errors.Wrap(err, "creating view") @@ -1064,7 +1064,7 @@ type Field struct { Max int64 `json:"max,omitempty"` } -// BitDepth returns the number of columns required to store a value between min & max. +// BitDepth returns the number of bits required to store a value between min & max. func (f *Field) BitDepth() uint { for i := uint(0); i < 63; i++ { if f.Max-f.Min < (1 << i) { diff --git a/handler_test.go b/handler_test.go index 94ff9b309..85bf152d6 100644 --- a/handler_test.go +++ b/handler_test.go @@ -1110,7 +1110,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { s.Handler.API.Holder = hldr.Holder defer s.Close() - // Set columns in the index. + // Set bits in the index. f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) f0.MustSetBits(100, 1, 2, 3) diff --git a/stats_test.go b/stats_test.go index bf4ff14db..5e88932d2 100644 --- a/stats_test.go +++ b/stats_test.go @@ -162,8 +162,8 @@ func TestStatsCount_SetColumnAttrs(t *testing.T) { frame.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { - if name != "SetColumnAttrs" { - t.Errorf("Expected SetColumnAttrs, Results %s", name) + if name != "SetRowAttrs" { + t.Errorf("Expected SetRowAttrs, Results %s", name) } called = true }, diff --git a/view.go b/view.go index 41a5ea287..e825aa8ef 100644 --- a/view.go +++ b/view.go @@ -305,7 +305,7 @@ func (v *View) DeleteFragment(slice uint64) error { return nil } -// SetBit sets a column within the view. +// SetBit sets a bit within the view. func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) @@ -315,7 +315,7 @@ func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { return frag.SetBit(rowID, columnID) } -// ClearBit clears a column within the view. +// ClearBit clears a bit within the view. func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) @@ -325,30 +325,30 @@ func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { return frag.ClearBit(rowID, columnID) } -// FieldValue uses a column of columns to read a multi-column value. -func (v *View) FieldValue(columnID uint64, columnDepth uint) (value uint64, exists bool, err error) { +// FieldValue uses a column of bits to read a multi-bit value. +func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return value, exists, err } - return frag.FieldValue(columnID, columnDepth) + return frag.FieldValue(columnID, bitDepth) } -// SetFieldValue uses a column of columns to set a multi-column value. -func (v *View) SetFieldValue(columnID uint64, columnDepth uint, value uint64) (changed bool, err error) { +// SetFieldValue uses a column of bits to set a multi-bit value. +func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) { slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.SetFieldValue(columnID, columnDepth, value) + return frag.SetFieldValue(columnID, bitDepth, value) } // FieldSum returns the sum & count of a field. -func (v *View) FieldSum(filter *Row, columnDepth uint) (sum, count uint64, err error) { +func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) { for _, f := range v.Fragments() { - fsum, fcount, err := f.FieldSum(filter, columnDepth) + fsum, fcount, err := f.FieldSum(filter, bitDepth) if err != nil { return sum, count, err } @@ -359,10 +359,10 @@ func (v *View) FieldSum(filter *Row, columnDepth uint) (sum, count uint64, err e } // FieldMin returns the min and count of a field. -func (v *View) FieldMin(filter *Row, columnDepth uint) (min, count uint64, err error) { +func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) { var minHasValue bool for _, f := range v.Fragments() { - fmin, fcount, err := f.FieldMin(filter, columnDepth) + fmin, fcount, err := f.FieldMin(filter, bitDepth) if err != nil { return min, count, err } @@ -387,9 +387,9 @@ func (v *View) FieldMin(filter *Row, columnDepth uint) (min, count uint64, err e } // FieldMax returns the max and count of a field. -func (v *View) FieldMax(filter *Row, columnDepth uint) (max, count uint64, err error) { +func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) { for _, f := range v.Fragments() { - fmax, fcount, err := f.FieldMax(filter, columnDepth) + fmax, fcount, err := f.FieldMax(filter, bitDepth) if err != nil { return max, count, err } @@ -402,10 +402,10 @@ func (v *View) FieldMax(filter *Row, columnDepth uint) (max, count uint64, err e } // FieldRange returns rows with a field value encoding matching the predicate. -func (v *View) FieldRange(op pql.Token, columnDepth uint, predicate uint64) (*Row, error) { +func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { - other, err := frag.FieldRange(op, columnDepth, predicate) + other, err := frag.FieldRange(op, bitDepth, predicate) if err != nil { return nil, err } @@ -416,10 +416,10 @@ func (v *View) FieldRange(op pql.Token, columnDepth uint, predicate uint64) (*Ro // FieldRangeBetween returns bitmaps with a field value encoding matching any // value between predicateMin and predicateMax. -func (v *View) FieldRangeBetween(columnDepth uint, predicateMin, predicateMax uint64) (*Row, error) { +func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) { r := NewRow() for _, frag := range v.Fragments() { - other, err := frag.FieldRangeBetween(columnDepth, predicateMin, predicateMax) + other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax) if err != nil { return nil, err } diff --git a/view_test.go b/view_test.go index f1c875c13..21834113d 100644 --- a/view_test.go +++ b/view_test.go @@ -83,7 +83,7 @@ func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { } // MustClearColumns clears columns on a row. Panic on error. -func (v *View) MustClearColumns(rowID uint64, columnIDs ...uint64) { +func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { if _, err := v.ClearBit(rowID, columnID); err != nil { panic(err) From b878ab347ad68051ca402b45663178ab27815ad8 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 25 May 2018 10:23:14 -0500 Subject: [PATCH 06/13] few more fixes to bsi comments --- fragment.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/fragment.go b/fragment.go index 9aa5e17e7..f4e9906a1 100644 --- a/fragment.go +++ b/fragment.go @@ -698,7 +698,7 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) { // Start with set of columns with values set. b := f.Row(uint64(bitDepth)) - // Filter any columns that don't match the current column value. + // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) bit := (predicate >> uint(i)) & 1 @@ -735,7 +735,7 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b // Start with set of columns with values set. b := f.Row(uint64(bitDepth)) - // Filter any columns that don't match the current column value. + // Filter any bits that don't match the current bit value. leadingZeros := true for i := int(bitDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) @@ -823,18 +823,18 @@ func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax u keep1 := NewRow() // GTE keep2 := NewRow() // LTE - // Filter any columns that don't match the current column value. + // Filter any bits that don't match the current bit value. for i := int(bitDepth - 1); i >= 0; i-- { row := f.Row(uint64(i)) bit1 := (predicateMin >> uint(i)) & 1 bit2 := (predicateMax >> uint(i)) & 1 // GTE predicateMin - // If bit is set then remove all unset bits not already kept. + // If bit is set then remove all unset columns not already kept. if bit1 == 1 { b = b.Difference(b.Difference(row).Difference(keep1)) } else { - // If bit is unset then add bits with set bit to keep. + // If bit is unset then add columns with set bit to keep. // Don't bother to compute this on the final iteration. if i > 0 { keep1 = keep1.Union(b.Intersect(row)) @@ -846,7 +846,7 @@ func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax u if bit2 == 0 { b = b.Difference(row.Difference(keep2)) } else { - // If bit is set then add bits for set bits to exclude. + // If bit is set then add columns for set bits to exclude. // Don't bother to compute this on the final iteration. if i > 0 { keep2 = keep2.Union(b.Difference(row)) From ca530b1fc39d536fcf5e2f37050f684cc84277d7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 25 May 2018 10:56:47 -0500 Subject: [PATCH 07/13] more fixes --- fragment_test.go | 12 ++++++------ holder.go | 2 +- holder_test.go | 2 +- row.go | 17 +++++++++-------- 4 files changed, 17 insertions(+), 16 deletions(-) diff --git a/fragment_test.go b/fragment_test.go index 6b6294d76..5c5f03b16 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -448,28 +448,28 @@ func TestFragment_FieldRange(t *testing.T) { t.Fatal(err) } - // Query for fields greater than (ending with unset column). + // Query for fields greater than (ending with unset bit). if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set column). + // Query for fields greater than (ending with set bit). if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset column). + // Query for fields greater than or equal to (ending with unset bit). if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set column). + // Query for fields greater than or equal to (ending with set bit). if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { @@ -838,7 +838,7 @@ func TestFragment_Blocks(t *testing.T) { // Retrieve initial checksum. var prev []pilosa.FragmentBlock - // Set first column. + // Set first bit. if _, err := f.SetBit(0, 0); err != nil { t.Fatal(err) } @@ -976,7 +976,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { f0 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "") defer f0.Close() - // Set and then clear columns on the fragment. + // Set and then clear bits on the fragment. if _, err := f0.SetBit(1000, 1); err != nil { t.Fatal(err) } else if _, err := f0.SetBit(1000, 2); err != nil { diff --git a/holder.go b/holder.go index 1d6b4f461..a80ba4539 100644 --- a/holder.go +++ b/holder.go @@ -466,7 +466,7 @@ func (h *Holder) flushCaches() { // RecalculateCaches recalculates caches on every index in the holder. This is // probably not practical to call in real-world workloads, but makes writing // integration tests much eaiser, since one doesn't have to wait 10 seconds -// after setting columns to get expected response. +// after setting bits to get expected response. func (h *Holder) RecalculateCaches() { for _, index := range h.Indexes() { index.RecalculateCaches() diff --git a/holder_test.go b/holder_test.go index c5b2ad76d..c7877e3ba 100644 --- a/holder_test.go +++ b/holder_test.go @@ -330,7 +330,7 @@ func TestHolder_DeleteIndex(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Write columns to separate indexes. + // Write bits to separate indexes. f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0) if _, err := f0.SetBit(100, 200); err != nil { t.Fatal(err) diff --git a/row.go b/row.go index 61e735626..ec2c42726 100644 --- a/row.go +++ b/row.go @@ -22,7 +22,8 @@ import ( "github.com/pilosa/pilosa/roaring" ) -// Row represents a set of columns. +// Row is a set of integers (the associated columns), and attributes which are +// arbitrary key/value pairs storing metadata about what the row represents. type Row struct { segments []RowSegment @@ -115,7 +116,7 @@ func (r *Row) Xor(other *Row) *Row { return &Row{segments: segments} } -// Union returns the columnwise union of r and other. +// Union returns the bitwise union of r and other. func (r *Row) Union(other *Row) *Row { var segments []RowSegment itr := newMergeSegmentIterator(r.segments, other.segments) @@ -226,7 +227,7 @@ func (r *Row) DecrementCount(i uint64) { } } -// Count returns the number of set columns in the row. +// Count returns the number of columns in the row. func (r *Row) Count() uint64 { var n uint64 for i := range r.segments { @@ -238,8 +239,8 @@ func (r *Row) Count() uint64 { // MarshalJSON returns a JSON-encoded byte slice of r. func (r *Row) MarshalJSON() ([]byte, error) { var o struct { - Attrs map[string]interface{} `json:"attrs"` - Columns []uint64 `json:"columns"` + Attrs map[string]interface{} `json:"attrs"` + Columns []uint64 `json:"columns"` } o.Columns = r.Columns() @@ -267,8 +268,8 @@ func encodeRow(r *Row) *internal.Row { } return &internal.Row{ - Columns: r.Columns(), - Attrs: encodeAttrs(r.Attrs), + Columns: r.Columns(), + Attrs: encodeAttrs(r.Attrs), } } @@ -339,7 +340,7 @@ func (s *RowSegment) Intersect(other *RowSegment) *RowSegment { } } -// Union returns the columnwise union of s and other. +// Union returns the bitwise union of s and other. func (s *RowSegment) Union(other *RowSegment) *RowSegment { data := s.data.Union(&other.data) From 1317d5e989035fdbfded02bddb7a3908aa904863 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 25 May 2018 11:02:14 -0500 Subject: [PATCH 08/13] more bit/column comment tweaks --- stats_test.go | 2 +- test/fragment.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/stats_test.go b/stats_test.go index 5e88932d2..611b7b2b6 100644 --- a/stats_test.go +++ b/stats_test.go @@ -78,7 +78,7 @@ func TestMultiStatClient_Expvar(t *testing.T) { t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String()) } - // Expvar should ignore earlier set tags from setcolumn + // Expvar should ignore earlier set tags from setbit if hldr.Stats.Tags() != nil { t.Fatalf("unexpected tag") } diff --git a/test/fragment.go b/test/fragment.go index 1018ef6c0..cd1111750 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -126,7 +126,7 @@ func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { s.attrs[id] = m } -// GenerateImportFill generates a set of columns pairs that evenly fill a fragment chunk. +// GenerateImportFill generates a set of row/col pairs that evenly fill a fragment chunk. func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { ipct := int(pct * 100) for i := 0; i < SliceWidth*rowN; i++ { From 75e8a873b1ce478402b3afd79fb704ebf46aed56 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 25 May 2018 13:09:32 -0500 Subject: [PATCH 09/13] remove rangeEnabled option everywhere --- cmd/import.go | 1 - docs/administration.md | 2 +- docs/api-reference.md | 1 - executor_test.go | 1 - frame.go | 1 - index.go | 5 -- index_test.go | 7 +- internal/private.pb.go | 178 ++++++++++++++++------------------------- internal/private.proto | 3 +- 9 files changed, 72 insertions(+), 127 deletions(-) diff --git a/cmd/import.go b/cmd/import.go index c6cad303a..8dd31181c 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -61,7 +61,6 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") - flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.") flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked") flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame") ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify) diff --git a/docs/administration.md b/docs/administration.md index 5a469c85e..d25f8a8d1 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -65,7 +65,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count ```
-

Note that you must first create a frame with range-encoding enabled and a field. View Create Frame for more details.

+

Note that you must first create a frame and a field. View Create Frame for more details.

#### Exporting diff --git a/docs/api-reference.md b/docs/api-reference.md index 7841dff2e..f94cc5009 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -105,7 +105,6 @@ The request payload is in JSON, and may contain the `options` field. The `option * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. -* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields. * `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding). Each individual `field` contains the following: diff --git a/executor_test.go b/executor_test.go index f2975548a..c6d2c6dda 100644 --- a/executor_test.go +++ b/executor_test.go @@ -609,7 +609,6 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100}, }, diff --git a/frame.go b/frame.go index de9d3527f..52fa2e752 100644 --- a/frame.go +++ b/frame.go @@ -1004,7 +1004,6 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { InverseEnabled bool `json:"inverseEnabled,omitempty"` - RangeEnabled bool `json:"rangeEnabled,omitempty"` // deprecated, will be removed CacheType string `json:"cacheType,omitempty"` CacheSize uint32 `json:"cacheSize,omitempty"` TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` diff --git a/index.go b/index.go index 4212576f6..0d8825234 100644 --- a/index.go +++ b/index.go @@ -325,11 +325,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, ErrInvalidCacheType } - // Validate mutually exclusive options if ranges are enabled. - if opt.RangeEnabled { - i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame") - } - // Validate fields. for _, field := range opt.Fields { if err := ValidateField(field); err != nil { diff --git a/index_test.go b/index_test.go index a83fb7609..ea0747060 100644 --- a/index_test.go +++ b/index_test.go @@ -67,14 +67,13 @@ func TestIndex_CreateFrame(t *testing.T) { }) // Ensure frame can include range columns. - t.Run("RangeEnabled", func(t *testing.T) { + t.Run("BSIFields", func(t *testing.T) { t.Run("OK", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() // Create frame with schema and verify it exists. if f, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: false, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, @@ -104,7 +103,6 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() frame, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, InverseEnabled: true, Fields: []*pilosa.Field{ &pilosa.Field{ @@ -153,7 +151,7 @@ func TestIndex_CreateFrame(t *testing.T) { } }) - t.Run("RangeEnabledWithCacheTypeNone", func(t *testing.T) { + t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ @@ -208,7 +206,6 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, }, diff --git a/internal/private.pb.go b/internal/private.pb.go index d7922711d..4400677c3 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -74,7 +74,6 @@ type FrameMeta struct { CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - RangeEnabled bool `protobuf:"varint,6,opt,name=RangeEnabled,proto3" json:"RangeEnabled,omitempty"` Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"` } @@ -111,13 +110,6 @@ func (m *FrameMeta) GetTimeQuantum() string { return "" } -func (m *FrameMeta) GetRangeEnabled() bool { - if m != nil { - return m.RangeEnabled - } - return false -} - func (m *FrameMeta) GetFields() []*Field { if m != nil { return m.Fields @@ -1094,16 +1086,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } - if m.RangeEnabled { - dAtA[i] = 0x30 - i++ - if m.RangeEnabled { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } if len(m.Fields) > 0 { for _, msg := range m.Fields { dAtA[i] = 0x3a @@ -2338,9 +2320,6 @@ func (m *FrameMeta) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.RangeEnabled { - n += 2 - } if len(m.Fields) > 0 { for _, e := range m.Fields { l = e.Size() @@ -3054,26 +3033,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RangeEnabled", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.RangeEnabled = bool(v != 0) case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) @@ -7259,75 +7218,74 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1112 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6f, 0x1b, 0x45, + // 1099 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, 0x14, 0x67, 0xbd, 0x6b, 0x27, 0x7e, 0xae, 0x53, 0x67, 0x5a, 0xca, 0xb6, 0xaa, 0x82, 0x19, 0x15, 0x6a, 0x38, 0x44, 0x25, 0xbd, 0x40, 0xa1, 0x52, 0x95, 0x38, 0x15, 0x8b, 0x48, 0x04, 0xe3, 0xa4, - 0x07, 0x24, 0x90, 0x26, 0xf6, 0x28, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0xf9, 0xe8, 0x81, 0x33, - 0x17, 0xee, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x01, 0x42, 0xe1, 0x0f, 0x01, 0xbd, 0x37, 0xb3, 0x1f, - 0xb1, 0x9d, 0xa6, 0x04, 0x6e, 0xf3, 0x3e, 0xe7, 0xf7, 0x3e, 0x67, 0x17, 0xda, 0x93, 0x34, 0x3c, - 0x96, 0x5a, 0xad, 0x4f, 0xd2, 0x44, 0x27, 0x6c, 0x39, 0x8c, 0xb5, 0x4a, 0x63, 0x19, 0xf1, 0x16, - 0x34, 0x83, 0x78, 0xa4, 0x4e, 0x77, 0x94, 0x96, 0xfc, 0x0f, 0x07, 0x9a, 0xcf, 0x53, 0x39, 0x56, - 0x48, 0xb1, 0x0f, 0x60, 0x25, 0x88, 0x8f, 0x55, 0x9a, 0xa9, 0xed, 0x58, 0x1e, 0x44, 0x6a, 0xe4, - 0xd7, 0xba, 0x4e, 0x6f, 0x59, 0xcc, 0x70, 0xd9, 0x7d, 0x68, 0x6e, 0xc9, 0xe1, 0x4b, 0xb5, 0x77, - 0x36, 0x51, 0xbe, 0xdb, 0x75, 0x7a, 0x4d, 0x51, 0x32, 0x0a, 0xe9, 0x20, 0x7c, 0xa5, 0x7c, 0xaf, - 0xeb, 0xf4, 0xda, 0xa2, 0x64, 0xb0, 0x2e, 0xb4, 0xf6, 0xc2, 0xb1, 0xfa, 0x66, 0x2a, 0x63, 0x3d, - 0x1d, 0xfb, 0x75, 0xb2, 0xae, 0xb2, 0x18, 0x87, 0x1b, 0x42, 0xc6, 0x87, 0x05, 0x86, 0x06, 0x61, - 0xb8, 0xc0, 0x63, 0x0f, 0xa1, 0xf1, 0x3c, 0x54, 0xd1, 0x28, 0xf3, 0x97, 0xba, 0x6e, 0xaf, 0xb5, - 0x71, 0x73, 0x3d, 0x8f, 0x6f, 0x9d, 0xf8, 0xc2, 0x8a, 0x39, 0x87, 0x95, 0x60, 0x3c, 0x49, 0x52, - 0x2d, 0x54, 0x36, 0x49, 0xe2, 0x4c, 0xb1, 0x0e, 0xb8, 0xdb, 0x69, 0xea, 0x3b, 0x74, 0x31, 0x1e, - 0xf9, 0x8f, 0xd0, 0xd9, 0x8c, 0x92, 0xe1, 0x51, 0x5f, 0x6a, 0x29, 0xd4, 0x0f, 0x53, 0x95, 0x69, - 0x76, 0x1b, 0xea, 0x94, 0x25, 0xab, 0x67, 0x08, 0xe4, 0x52, 0xb6, 0x28, 0x2f, 0x4d, 0x61, 0x08, - 0xe4, 0x92, 0x3d, 0xa5, 0xc2, 0x13, 0x86, 0x40, 0xee, 0x20, 0x0a, 0x87, 0x26, 0x05, 0x9e, 0x30, - 0x04, 0x63, 0xe0, 0xbd, 0x08, 0xd5, 0x89, 0x8d, 0x9b, 0xce, 0x3c, 0x80, 0xd5, 0xca, 0xfd, 0x16, - 0xe6, 0x1d, 0x68, 0x88, 0xe4, 0x24, 0xe8, 0x67, 0xbe, 0xd3, 0x75, 0x7b, 0x9e, 0xb0, 0x14, 0x65, - 0x37, 0x89, 0xa6, 0xe3, 0x18, 0x45, 0x35, 0x12, 0x95, 0x0c, 0x7e, 0x17, 0xea, 0x94, 0x6a, 0x8c, - 0xb2, 0xb4, 0xc5, 0x23, 0xff, 0xdb, 0x81, 0xe6, 0x8e, 0x3c, 0x25, 0x18, 0x19, 0x7b, 0x0a, 0xcb, - 0x03, 0x2d, 0xe3, 0x91, 0x4c, 0x47, 0xa4, 0xd4, 0xda, 0x78, 0xaf, 0x4c, 0x61, 0xa1, 0xb6, 0x9e, - 0xeb, 0x6c, 0xc7, 0x3a, 0x3d, 0x13, 0x85, 0x09, 0x7b, 0x02, 0x4b, 0xb6, 0x27, 0x08, 0x43, 0x6b, - 0xa3, 0xbb, 0xc8, 0xba, 0x68, 0x1b, 0x34, 0xce, 0x0d, 0xee, 0x7d, 0x06, 0xed, 0x0b, 0x6e, 0x11, - 0xeb, 0x91, 0x3a, 0xcb, 0x2b, 0x72, 0xa4, 0xce, 0x30, 0x77, 0xc7, 0x32, 0x9a, 0x9a, 0x3c, 0x7b, - 0xc2, 0x10, 0x4f, 0x6a, 0x9f, 0x38, 0xf7, 0x9e, 0xc0, 0x8d, 0xaa, 0xd7, 0x7f, 0x63, 0xcb, 0xbf, - 0x07, 0xb6, 0x95, 0x2a, 0xa9, 0x15, 0xc1, 0xdb, 0x51, 0x59, 0x26, 0x0f, 0xd5, 0xe5, 0x95, 0x36, - 0xd5, 0xab, 0x55, 0xab, 0x77, 0x1f, 0x9a, 0x41, 0x96, 0x07, 0xee, 0x52, 0x5f, 0x96, 0x0c, 0xfe, - 0x11, 0xb0, 0xbe, 0x8a, 0x94, 0x56, 0x76, 0xbe, 0x5e, 0xe3, 0x9f, 0x0f, 0x72, 0x2c, 0x57, 0xeb, - 0xb2, 0x87, 0xe0, 0xe1, 0x78, 0x12, 0x94, 0xd6, 0xc6, 0xad, 0x32, 0xd3, 0xc5, 0x1c, 0x0b, 0x52, - 0xe0, 0x61, 0xee, 0xd4, 0x8e, 0xf4, 0x15, 0x01, 0x2e, 0x68, 0xe5, 0xfc, 0x2a, 0x77, 0xf6, 0xaa, - 0x62, 0x49, 0xd8, 0xab, 0x9e, 0xe5, 0xb1, 0x5e, 0xf7, 0x2a, 0x7e, 0x58, 0x80, 0xc5, 0x49, 0xbd, - 0x0e, 0xd8, 0xf7, 0xa1, 0x4e, 0xb6, 0x16, 0xed, 0xdc, 0x0e, 0x30, 0x52, 0xfe, 0xa2, 0x80, 0x7a, - 0xdd, 0x8b, 0x6e, 0x57, 0x2f, 0x6a, 0xe6, 0x7e, 0xbf, 0xb5, 0xba, 0x38, 0xd3, 0xbb, 0x68, 0x63, - 0x3c, 0xd1, 0xf9, 0xf2, 0x9a, 0xcd, 0x24, 0x12, 0x7d, 0xe3, 0x12, 0xc8, 0x7c, 0xb7, 0xeb, 0xa2, - 0x6f, 0x22, 0xf8, 0x63, 0x68, 0x0c, 0x86, 0x2f, 0xd5, 0x58, 0xb2, 0x0f, 0x71, 0xd2, 0x46, 0xea, - 0x54, 0x65, 0x76, 0x4e, 0x6f, 0xce, 0xd4, 0x5f, 0xe4, 0x72, 0xde, 0xb7, 0x21, 0x5d, 0x02, 0xa8, - 0x41, 0x57, 0x67, 0xbe, 0x37, 0xb7, 0x31, 0x91, 0x2f, 0xac, 0x98, 0x6f, 0x83, 0xbb, 0x2f, 0x02, - 0xdc, 0x3f, 0x84, 0x20, 0xf7, 0x62, 0x29, 0xf4, 0xfd, 0x45, 0x92, 0x69, 0x9b, 0x20, 0x3a, 0x23, - 0xef, 0xeb, 0x24, 0xd5, 0x94, 0x9e, 0xb6, 0xa0, 0x33, 0xff, 0x0e, 0xbc, 0xdd, 0x64, 0xa4, 0xd8, - 0x0a, 0xd4, 0x82, 0xbe, 0xf5, 0x51, 0x0b, 0xfa, 0xec, 0x5d, 0x72, 0x6f, 0xf3, 0xd2, 0x2e, 0x41, - 0xec, 0x8b, 0x40, 0xd0, 0xc5, 0x0f, 0xa0, 0x1d, 0x64, 0x5b, 0x49, 0x92, 0x8e, 0xc2, 0x58, 0xea, - 0x24, 0xb5, 0x73, 0x76, 0x91, 0xc9, 0x9f, 0x41, 0x07, 0xdd, 0x0f, 0xb4, 0xd4, 0x45, 0xf7, 0xdd, - 0x81, 0x06, 0xf2, 0x8a, 0xeb, 0x2c, 0x45, 0xb3, 0x8c, 0x7a, 0x79, 0x51, 0x89, 0xe0, 0x5f, 0x19, - 0x0f, 0xdb, 0xc7, 0x2a, 0xd6, 0x95, 0xa6, 0x20, 0x9a, 0x1c, 0xb4, 0x85, 0x21, 0x18, 0x37, 0xa1, - 0x58, 0xcc, 0x2b, 0x25, 0x66, 0xe4, 0x0a, 0x92, 0xf1, 0x9f, 0x1d, 0x80, 0x1c, 0xd0, 0x34, 0x2b, - 0x4c, 0x9c, 0xcb, 0x4d, 0xd8, 0xc7, 0x95, 0x7d, 0x3c, 0xdf, 0x27, 0x85, 0x48, 0x54, 0xb6, 0x76, - 0x2f, 0x6f, 0x0b, 0xdb, 0xf2, 0x9d, 0x52, 0xdf, 0xf0, 0x6d, 0x99, 0x70, 0x15, 0xb4, 0xb7, 0xa2, - 0x69, 0xa6, 0x55, 0x6a, 0x11, 0xe1, 0xbb, 0x61, 0x18, 0x45, 0x7e, 0x4a, 0xc6, 0xe2, 0x14, 0xb1, - 0x07, 0x50, 0x47, 0xa4, 0xa6, 0x37, 0xe7, 0xc3, 0x30, 0x42, 0x3e, 0xb0, 0xd3, 0xb1, 0xb0, 0xed, - 0x18, 0x78, 0xf4, 0x95, 0x60, 0xdb, 0x85, 0x3e, 0x10, 0x3a, 0xe0, 0xee, 0x84, 0x31, 0x85, 0xe0, - 0x0a, 0x3c, 0x12, 0x47, 0x9e, 0xd2, 0x4b, 0x89, 0x1c, 0x89, 0xfb, 0x71, 0xd5, 0x6c, 0x07, 0x9c, - 0x87, 0xeb, 0xcc, 0x6c, 0xfe, 0xd0, 0xba, 0x95, 0x87, 0x76, 0x00, 0xab, 0x66, 0x13, 0xfc, 0x9f, - 0x4e, 0x7f, 0xad, 0xc1, 0xaa, 0x50, 0x59, 0xf8, 0x4a, 0x05, 0x71, 0xa6, 0xd3, 0xe9, 0x50, 0x87, - 0x49, 0x8c, 0xf6, 0x5f, 0x26, 0x07, 0x36, 0xd5, 0xae, 0x30, 0xc4, 0x9b, 0x74, 0x12, 0x7b, 0x04, - 0xad, 0xd9, 0xee, 0x9f, 0x57, 0xad, 0xaa, 0xb0, 0x47, 0xb0, 0x34, 0x48, 0xa6, 0xe9, 0xb0, 0x98, - 0xed, 0x3b, 0xa5, 0xb6, 0x41, 0x66, 0xc4, 0x22, 0x57, 0xab, 0xf4, 0x51, 0xfd, 0xf5, 0x7d, 0xc4, - 0x9e, 0xce, 0xf4, 0x11, 0x7d, 0x8d, 0xb5, 0x36, 0xde, 0x29, 0x0d, 0x2e, 0x88, 0xc5, 0x45, 0x6d, - 0xfe, 0x93, 0x03, 0x37, 0xaa, 0x10, 0xde, 0x68, 0x30, 0x8a, 0x8a, 0xd4, 0x16, 0x56, 0xc4, 0x5d, - 0x54, 0x11, 0xaf, 0xac, 0x48, 0xf9, 0x76, 0xd7, 0x2b, 0x6f, 0x37, 0x3f, 0x82, 0xbb, 0x73, 0x65, - 0xda, 0x4a, 0xc6, 0x13, 0xec, 0x87, 0xff, 0x50, 0x2e, 0x5c, 0x19, 0x69, 0x6a, 0x0b, 0xd5, 0x14, - 0x86, 0xe0, 0x9f, 0xc2, 0xdb, 0x03, 0xa5, 0x2b, 0x45, 0xca, 0xbb, 0xad, 0x0b, 0xee, 0xae, 0x3a, - 0xb9, 0x24, 0x7c, 0x14, 0xf1, 0xcf, 0xc1, 0xdf, 0x9f, 0x8c, 0xa4, 0x56, 0xd7, 0xb2, 0xde, 0x84, - 0xe5, 0xbd, 0x64, 0x92, 0x44, 0xc9, 0xe1, 0xd9, 0x15, 0x23, 0xef, 0xc3, 0x92, 0xd9, 0x8f, 0xe6, - 0x33, 0xb2, 0x29, 0x72, 0x92, 0xdf, 0xc2, 0x86, 0x1e, 0xca, 0x68, 0x38, 0x8d, 0x10, 0x06, 0x7e, - 0x4f, 0x66, 0x9b, 0x9d, 0xdf, 0xce, 0xd7, 0x9c, 0xdf, 0xcf, 0xd7, 0x9c, 0x3f, 0xcf, 0xd7, 0x9c, - 0x5f, 0xfe, 0x5a, 0x7b, 0xeb, 0xa0, 0x41, 0x7f, 0x16, 0x8f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, - 0xfa, 0xf5, 0x5b, 0x36, 0x6a, 0x0c, 0x00, 0x00, + 0x07, 0x24, 0x90, 0x26, 0xf6, 0x28, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0xf9, 0xd3, 0x03, 0x67, + 0x2e, 0xdc, 0x11, 0x1f, 0x85, 0x4f, 0xc0, 0x91, 0x8f, 0x80, 0xc2, 0x07, 0x01, 0xbd, 0x37, 0xb3, + 0x7f, 0x62, 0x3b, 0x4d, 0x09, 0xdc, 0xe6, 0xfd, 0x9d, 0xdf, 0xfb, 0x3b, 0xbb, 0xd0, 0x9e, 0xa4, + 0xe1, 0xb1, 0xd4, 0x6a, 0x7d, 0x92, 0x26, 0x3a, 0x61, 0xcb, 0x61, 0xac, 0x55, 0x1a, 0xcb, 0x88, + 0xb7, 0xa0, 0x19, 0xc4, 0x23, 0x75, 0xba, 0xa3, 0xb4, 0xe4, 0xbf, 0x39, 0xd0, 0x7c, 0x9e, 0xca, + 0xb1, 0x42, 0x8a, 0x7d, 0x00, 0x2b, 0x41, 0x7c, 0xac, 0xd2, 0x4c, 0x6d, 0xc7, 0xf2, 0x20, 0x52, + 0x23, 0xbf, 0xd6, 0x75, 0x7a, 0xcb, 0x62, 0x86, 0xcb, 0xee, 0x43, 0x73, 0x4b, 0x0e, 0x5f, 0xaa, + 0xbd, 0xb3, 0x89, 0xf2, 0xdd, 0xae, 0xd3, 0x6b, 0x8a, 0x92, 0x51, 0x48, 0x07, 0xe1, 0x2b, 0xe5, + 0x7b, 0x5d, 0xa7, 0xd7, 0x16, 0x25, 0x83, 0x75, 0xa1, 0xb5, 0x17, 0x8e, 0xd5, 0x37, 0x53, 0x19, + 0xeb, 0xe9, 0xd8, 0xaf, 0x93, 0x75, 0x95, 0xc5, 0x1e, 0x42, 0xe3, 0x79, 0xa8, 0xa2, 0x51, 0xe6, + 0x2f, 0x75, 0xdd, 0x5e, 0x6b, 0xe3, 0xe6, 0x7a, 0x8e, 0x7d, 0x9d, 0xf8, 0xc2, 0x8a, 0x39, 0x87, + 0x95, 0x60, 0x3c, 0x49, 0x52, 0x2d, 0x54, 0x36, 0x49, 0xe2, 0x4c, 0xb1, 0x0e, 0xb8, 0xdb, 0x69, + 0xea, 0x3b, 0xe4, 0x14, 0x8f, 0xfc, 0x47, 0xe8, 0x6c, 0x46, 0xc9, 0xf0, 0xa8, 0x2f, 0xb5, 0x14, + 0xea, 0x87, 0xa9, 0xca, 0x34, 0xbb, 0x0d, 0x75, 0xca, 0x80, 0xd5, 0x33, 0x04, 0x72, 0x29, 0x13, + 0x14, 0x73, 0x53, 0x18, 0x02, 0xb9, 0x64, 0x4f, 0x61, 0x7a, 0xc2, 0x10, 0xc8, 0x1d, 0x44, 0xe1, + 0xd0, 0x84, 0xe7, 0x09, 0x43, 0x30, 0x06, 0xde, 0x8b, 0x50, 0x9d, 0xd8, 0x98, 0xe8, 0xcc, 0x03, + 0x58, 0xad, 0xdc, 0x6f, 0x61, 0xde, 0x81, 0x86, 0x48, 0x4e, 0x82, 0x7e, 0xe6, 0x3b, 0x5d, 0xb7, + 0xe7, 0x09, 0x4b, 0x51, 0xe6, 0x92, 0x68, 0x3a, 0x8e, 0x51, 0x54, 0x23, 0x51, 0xc9, 0xe0, 0x77, + 0xa1, 0x4e, 0x69, 0xc4, 0x28, 0x4b, 0x5b, 0x3c, 0xf2, 0xbf, 0x1d, 0x68, 0xee, 0xc8, 0x53, 0x82, + 0x91, 0xb1, 0xa7, 0xb0, 0x3c, 0xd0, 0x32, 0x1e, 0xc9, 0x74, 0x44, 0x4a, 0xad, 0x8d, 0xf7, 0xca, + 0x14, 0x16, 0x6a, 0xeb, 0xb9, 0xce, 0x76, 0xac, 0xd3, 0x33, 0x51, 0x98, 0xb0, 0x27, 0xb0, 0x64, + 0xeb, 0x4d, 0x18, 0x5a, 0x1b, 0xdd, 0x45, 0xd6, 0x45, 0x4b, 0xa0, 0x71, 0x6e, 0x70, 0xef, 0x33, + 0x68, 0x5f, 0x70, 0x8b, 0x58, 0x8f, 0xd4, 0x59, 0x5e, 0x91, 0x23, 0x75, 0x86, 0xb9, 0x3b, 0x96, + 0xd1, 0xd4, 0xe4, 0xd9, 0x13, 0x86, 0x78, 0x52, 0xfb, 0xc4, 0xb9, 0xf7, 0x04, 0x6e, 0x54, 0xbd, + 0xfe, 0x1b, 0x5b, 0xfe, 0x3d, 0xb0, 0xad, 0x54, 0x49, 0xad, 0x08, 0xde, 0x8e, 0xca, 0x32, 0x79, + 0xa8, 0x2e, 0xaf, 0xb4, 0xa9, 0x5e, 0xad, 0x5a, 0xbd, 0xfb, 0xd0, 0x0c, 0xb2, 0x3c, 0x70, 0x97, + 0xfa, 0xbe, 0x64, 0xf0, 0x8f, 0x80, 0xf5, 0x55, 0xa4, 0xb4, 0xb2, 0xb3, 0xf3, 0x1a, 0xff, 0x7c, + 0x90, 0x63, 0xb9, 0x5a, 0x97, 0x3d, 0x04, 0x0f, 0x47, 0x8f, 0xa0, 0xb4, 0x36, 0x6e, 0x95, 0x99, + 0x2e, 0x66, 0x54, 0x90, 0x02, 0x0f, 0x73, 0xa7, 0x76, 0x5c, 0xaf, 0x08, 0x70, 0x41, 0x2b, 0xe7, + 0x57, 0xb9, 0xb3, 0x57, 0x15, 0x0b, 0xc0, 0x5e, 0xf5, 0x2c, 0x8f, 0xf5, 0xba, 0x57, 0xf1, 0xc3, + 0x02, 0x2c, 0x4e, 0xea, 0x75, 0xc0, 0xbe, 0x0f, 0x75, 0xb2, 0xb5, 0x68, 0xe7, 0x76, 0x80, 0x91, + 0xf2, 0x17, 0x05, 0xd4, 0xeb, 0x5e, 0x74, 0xbb, 0x7a, 0x51, 0x33, 0xf7, 0xfb, 0xad, 0xd5, 0xc5, + 0x99, 0xde, 0x45, 0x1b, 0xe3, 0x89, 0xce, 0x97, 0xd7, 0x6c, 0x26, 0x91, 0xe8, 0x1b, 0x97, 0x40, + 0xe6, 0xbb, 0x5d, 0x17, 0x7d, 0x13, 0xc1, 0x1f, 0x43, 0x63, 0x30, 0x7c, 0xa9, 0xc6, 0x92, 0x7d, + 0x88, 0x93, 0x36, 0x52, 0xa7, 0x2a, 0xb3, 0x73, 0x7a, 0x73, 0xa6, 0xfe, 0x22, 0x97, 0xf3, 0xbe, + 0x0d, 0xe9, 0x12, 0x40, 0x0d, 0xba, 0x3a, 0xf3, 0xbd, 0xb9, 0x8d, 0x89, 0x7c, 0x61, 0xc5, 0x7c, + 0x1b, 0xdc, 0x7d, 0x11, 0xe0, 0xfe, 0x21, 0x04, 0xb9, 0x17, 0x4b, 0xa1, 0xef, 0x2f, 0x92, 0x4c, + 0xdb, 0x04, 0xd1, 0x19, 0x79, 0x5f, 0x27, 0xa9, 0xa6, 0xf4, 0xb4, 0x05, 0x9d, 0xf9, 0x77, 0xe0, + 0xed, 0x26, 0x23, 0xc5, 0x56, 0xa0, 0x16, 0xf4, 0xad, 0x8f, 0x5a, 0xd0, 0x67, 0xef, 0x92, 0x7b, + 0x9b, 0x97, 0x76, 0x09, 0x62, 0x5f, 0x04, 0x82, 0x2e, 0x7e, 0x00, 0xed, 0x20, 0xdb, 0x4a, 0x92, + 0x74, 0x14, 0xc6, 0x52, 0x27, 0xa9, 0x9d, 0xb3, 0x8b, 0x4c, 0xfe, 0x0c, 0x3a, 0xe8, 0x7e, 0xa0, + 0xa5, 0x2e, 0xba, 0xef, 0x0e, 0x34, 0x90, 0x57, 0x5c, 0x67, 0x29, 0x9a, 0x65, 0xd4, 0xcb, 0x8b, + 0x4a, 0x04, 0xff, 0xca, 0x78, 0xd8, 0x3e, 0x56, 0xb1, 0xae, 0x34, 0x05, 0xd1, 0xe4, 0xa0, 0x2d, + 0x0c, 0xc1, 0xb8, 0x09, 0xc5, 0x62, 0x5e, 0x29, 0x31, 0x23, 0x57, 0x90, 0x8c, 0xff, 0xec, 0x00, + 0xe4, 0x80, 0xa6, 0x59, 0x61, 0xe2, 0x5c, 0x6e, 0xc2, 0x3e, 0xae, 0xec, 0xe3, 0xf9, 0x3e, 0x29, + 0x44, 0xa2, 0xb2, 0xb5, 0x7b, 0x79, 0x5b, 0xd8, 0x96, 0xef, 0x94, 0xfa, 0x86, 0x6f, 0xcb, 0x84, + 0xab, 0xa0, 0xbd, 0x15, 0x4d, 0x33, 0xad, 0x52, 0x8b, 0x08, 0xdf, 0x0d, 0xc3, 0x28, 0xf2, 0x53, + 0x32, 0x16, 0xa7, 0x88, 0x3d, 0x80, 0x3a, 0x22, 0x35, 0xbd, 0x39, 0x1f, 0x86, 0x11, 0xf2, 0x81, + 0x9d, 0x8e, 0x85, 0x6d, 0xc7, 0xc0, 0xa3, 0x2f, 0x00, 0xdb, 0x2e, 0xf4, 0xf8, 0x77, 0xc0, 0xdd, + 0x09, 0x63, 0x0a, 0xc1, 0x15, 0x78, 0x24, 0x8e, 0x3c, 0xa5, 0x97, 0x12, 0x39, 0x12, 0xf7, 0xe3, + 0xaa, 0xd9, 0x0e, 0x38, 0x0f, 0xd7, 0x99, 0xd9, 0xfc, 0xa1, 0x75, 0x2b, 0x0f, 0xed, 0x00, 0x56, + 0xcd, 0x26, 0xf8, 0x3f, 0x9d, 0xfe, 0x5a, 0x83, 0x55, 0xa1, 0xb2, 0xf0, 0x95, 0x0a, 0xe2, 0x4c, + 0xa7, 0xd3, 0xa1, 0x0e, 0x93, 0x18, 0xed, 0xbf, 0x4c, 0x0e, 0x6c, 0xaa, 0x5d, 0x61, 0x88, 0x37, + 0xe9, 0x24, 0xf6, 0x08, 0x5a, 0xb3, 0xdd, 0x3f, 0xaf, 0x5a, 0x55, 0x61, 0x8f, 0x60, 0x69, 0x90, + 0x4c, 0xd3, 0x61, 0x31, 0xdb, 0x77, 0x4a, 0x6d, 0x83, 0xcc, 0x88, 0x45, 0xae, 0x56, 0xe9, 0xa3, + 0xfa, 0xeb, 0xfb, 0x88, 0x3d, 0x9d, 0xe9, 0x23, 0xbf, 0x41, 0x06, 0xef, 0x94, 0x06, 0x17, 0xc4, + 0xe2, 0xa2, 0x36, 0xff, 0xc9, 0x81, 0x1b, 0x55, 0x08, 0x6f, 0x34, 0x18, 0x45, 0x45, 0x6a, 0x0b, + 0x2b, 0xe2, 0x2e, 0xaa, 0x88, 0x57, 0x56, 0xa4, 0x7c, 0xbb, 0xeb, 0x95, 0xb7, 0x9b, 0x1f, 0xc1, + 0xdd, 0xb9, 0x32, 0x6d, 0x25, 0xe3, 0x09, 0xf6, 0xc3, 0x7f, 0x28, 0x17, 0xae, 0x8c, 0x34, 0xb5, + 0x85, 0x6a, 0x0a, 0x43, 0xf0, 0x4f, 0xe1, 0xed, 0x81, 0xd2, 0x95, 0x22, 0xe5, 0xdd, 0xd6, 0x05, + 0x77, 0x57, 0x9d, 0x5c, 0x12, 0x3e, 0x8a, 0xf8, 0xe7, 0xe0, 0xef, 0x4f, 0x46, 0x52, 0xab, 0x6b, + 0x59, 0x6f, 0xc2, 0xf2, 0x5e, 0x32, 0x49, 0xa2, 0xe4, 0xf0, 0xec, 0x8a, 0x91, 0xf7, 0x61, 0xc9, + 0xec, 0x47, 0xf3, 0x19, 0xd9, 0x14, 0x39, 0xc9, 0x6f, 0x61, 0x43, 0x0f, 0x65, 0x34, 0x9c, 0x46, + 0x08, 0x03, 0xbf, 0x27, 0xb3, 0xcd, 0xce, 0xef, 0xe7, 0x6b, 0xce, 0x1f, 0xe7, 0x6b, 0xce, 0x9f, + 0xe7, 0x6b, 0xce, 0x2f, 0x7f, 0xad, 0xbd, 0x75, 0xd0, 0xa0, 0xbf, 0x86, 0xc7, 0xff, 0x04, 0x00, + 0x00, 0xff, 0xff, 0x60, 0xc2, 0x0f, 0x6a, 0x46, 0x0c, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 563b57424..8efe9e0ad 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -10,8 +10,7 @@ message FrameMeta { string CacheType = 3; uint32 CacheSize = 4; string TimeQuantum = 5; - bool RangeEnabled = 6; - repeated Field Fields = 7; + repeated Field Fields = 7; } message ImportResponse { From 4767ea7bad6ff601c198ef2f2c49fe3006d9aba7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 25 May 2018 14:34:25 -0500 Subject: [PATCH 10/13] fix indentation issue --- internal/private.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/private.proto b/internal/private.proto index 8efe9e0ad..e1642698d 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -10,7 +10,7 @@ message FrameMeta { string CacheType = 3; uint32 CacheSize = 4; string TimeQuantum = 5; - repeated Field Fields = 7; + repeated Field Fields = 7; } message ImportResponse { From 47f7beaecc0ffbe1e5d670aa7feb30d6c2097dd7 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 25 May 2018 14:33:30 -0500 Subject: [PATCH 11/13] WIP removing inverse --- api.go | 7 - client.go | 17 +- client_test.go | 111 ------------ cluster.go | 13 +- cluster_test.go | 2 - cmd/export.go | 1 - cmd/export_test.go | 8 - ctl/export.go | 14 +- ctl/export_test.go | 8 - executor.go | 140 ++------------- executor_test.go | 43 +---- frame.go | 109 +++--------- handler.go | 4 +- handler_internal_test.go | 4 +- handler_test.go | 55 +----- holder.go | 10 -- index.go | 32 +--- index_test.go | 42 ----- internal/private.pb.go | 369 ++++++++------------------------------- internal/private.proto | 3 - pilosa.go | 7 +- pql/ast.go | 28 --- pql/ast_test.go | 66 ------- server.go | 26 +-- server/server_test.go | 52 +----- view.go | 13 +- webui/assets/main.js | 3 - 27 files changed, 151 insertions(+), 1036 deletions(-) diff --git a/api.go b/api.go index 7f8270de3..a2e2094c3 100644 --- a/api.go +++ b/api.go @@ -813,12 +813,6 @@ func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxSlices() } -// MaxInverseSlices returns the maximum inverse slice number for each index in a -// map. -func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 { - return api.Holder.MaxInverseSlices() -} - // StatsWithTags returns an instance of whatever implementation of StatsClient // pilosa is using with the given tags. func (api *API) StatsWithTags(tags []string) StatsClient { @@ -968,7 +962,6 @@ const ( //apiLocalID // not implemented //apiLongQueryTime // not implemented apiMarshalFragment - //apiMaxInverseSlices // not implemented //apiMaxSlices // not implemented apiQuery apiRecalculateCaches diff --git a/client.go b/client.go index a18d066f1..0b22aed81 100644 --- a/client.go +++ b/client.go @@ -77,16 +77,11 @@ func (c *InternalHTTPClient) Host() *URI { return c.defaultURI } // MaxSliceByIndex returns the number of slices on a server by index. func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx, false) -} - -// MaxInverseSliceByIndex returns the number of inverse slices on a server by index. -func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx, true) + return c.maxSliceByIndex(ctx) } // maxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { +func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { // Execute request against the host. u := uriPathToURL(c.defaultURI, "/slices/max") @@ -112,9 +107,6 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) return nil, fmt.Errorf("json decode: %s", err) } - if inverse { - return rsp.Inverse, nil - } return rsp.Standard, nil } @@ -524,7 +516,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s return ErrIndexRequired } else if frame == "" { return ErrFrameRequired - } else if !(view == ViewStandard || view == ViewInverse) { + } else if view != ViewStandard { return ErrInvalidView } @@ -605,8 +597,6 @@ func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, f var err error if view == ViewStandard { maxSlices, err = c.MaxSliceByIndex(ctx) - } else if view == ViewInverse { - maxSlices, err = c.MaxInverseSliceByIndex(ctx) } else { return ErrInvalidView } @@ -1315,7 +1305,6 @@ func nodePathToURL(node *Node, path string) url.URL { // I don't want to let it go unquestioned. type InternalClient interface { MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) - MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) diff --git a/client_test.go b/client_test.go index 960d93421..d8f1bf6d7 100644 --- a/client_test.go +++ b/client_test.go @@ -240,60 +240,6 @@ func TestClient_Import(t *testing.T) { } } -// Ensure client can bulk import data to an inverse frame. -func TestClient_ImportInverseEnabled(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frameOpts := pilosa.FrameOptions{ - InverseEnabled: true, - } - frame, err := idx.CreateFrameIfNotExists("f", frameOpts) - if err != nil { - panic(err) - } - v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse) - if err != nil { - panic(err) - } - f, err := v.CreateFragmentIfNotExists(0) - if err != nil { - panic(err) - } - - // Load bitmap into cache to ensure cache gets updated. - f.Row(0) - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) - if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ - {RowID: 0, ColumnID: 1}, - {RowID: 0, ColumnID: 5}, - {RowID: 200, ColumnID: 5}, - {RowID: 200, ColumnID: 6}, - }); err != nil { - t.Fatal(err) - } - - // Verify data. - if a := f.Row(1).Columns(); !reflect.DeepEqual(a, []uint64{0}) { - t.Fatalf("unexpected columns: %+v", a) - } - if a := f.Row(5).Columns(); !reflect.DeepEqual(a, []uint64{0, 200}) { - t.Fatalf("unexpected columns: %+v", a) - } - if a := f.Row(6).Columns(); !reflect.DeepEqual(a, []uint64{200}) { - t.Fatalf("unexpected columns: %+v", a) - } -} - // Ensure client can bulk import value data. func TestClient_ImportValue(t *testing.T) { hldr := test.MustOpenHolder() @@ -417,63 +363,6 @@ func TestClient_BackupRestore(t *testing.T) { } } -// Ensure client backup and restore a frame with inverse view. -func TestClient_BackupInverseView(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frameOpts := pilosa.FrameOptions{ - InverseEnabled: true, - } - frame, err := idx.CreateFrameIfNotExists("f", frameOpts) - if err != nil { - panic(err) - } - v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse) - if err != nil { - panic(err) - } - f, err := v.CreateFragmentIfNotExists(0) - if err != nil { - panic(err) - } - - f.SetBit(100, 1) - f.SetBit(100, 2) - f.SetBit(100, 3) - f.SetBit(100, SliceWidth-1) - - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - c := test.MustNewClient(s.Host(), defaultClient) - - // Backup from frame. - var buf bytes.Buffer - if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewInverse); err != nil { - t.Fatal(err) - } - - // Restore to a different frame. - if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{InverseEnabled: true}); err != nil { - t.Fatal(err) - } - if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewInverse); err != nil { - t.Fatal(err) - } - - // Verify data. - if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { - t.Fatalf("unexpected columns(0): %+v", a) - } - -} - // backup returns error with invalid view func TestClient_BackupInvalidView(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/cluster.go b/cluster.go index 97afa0eef..d6026274a 100644 --- a/cluster.go +++ b/cluster.go @@ -626,21 +626,14 @@ func (a viewsByFrame) addView(frame, view string) { func (c *Cluster) fragsByHost(idx *Index) fragsByHost { // frameViews is a map of frame to slice of views. frameViews := make(viewsByFrame) - inverseFrameViews := make(viewsByFrame) for _, frame := range idx.Frames() { for _, view := range frame.Views() { - if IsInverseView(view.Name()) { - inverseFrameViews.addView(frame.Name(), view.Name()) - } else { - frameViews.addView(frame.Name(), view.Name()) - } + frameViews.addView(frame.Name(), view.Name()) + } } - - std := c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews) - inv := c.fragCombos(idx.Name(), idx.MaxInverseSlice(), inverseFrameViews) - return std.add(inv) + return c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews) } // fragCombos returns a map (by uri) of lists of fragments for a given index diff --git a/cluster_test.go b/cluster_test.go index b09820ac8..913cd5db3 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -424,8 +424,6 @@ func TestCluster_ResizeStates(t *testing.T) { // Add Field Data to node0. if err := tc.CreateFrame("i", "fields", FrameOptions{ - InverseEnabled: false, - //CacheType: CacheTypeNone, Fields: []*Field{ { Name: "fld0", diff --git a/cmd/export.go b/cmd/export.go index c5907fbea..951c4a0a9 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -53,7 +53,6 @@ The file does not contain any headers. flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export") flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export") - flags.StringVarP(&Exporter.View, "view", "v", "standard", "View to export - default standard") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify) diff --git a/cmd/export_test.go b/cmd/export_test.go index 2d01cf4d6..2b30116c6 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -44,7 +44,6 @@ frame = "f1" v.Check(cmd.Exporter.Host, "localhost:12345") v.Check(cmd.Exporter.Index, "myindex") v.Check(cmd.Exporter.Frame, "f1") - v.Check(cmd.Exporter.View, "standard") v.Check(cmd.Exporter.Path, "/somefile") return v.Error() }, @@ -52,10 +51,3 @@ frame = "f1" } executeDry(t, tests) } - -func TestExportInvalidView(t *testing.T) { - output, err := ExecNewRootCommand(t, "export", "-i", "foo", "-f", "bar", "-v", "test") - if !strings.Contains(err.Error(), "invalid view") { - t.Fatalf("Command 'export' with invalid view should error but: err: '%v', output: '%v'", err, output) - } -} diff --git a/ctl/export.go b/ctl/export.go index 546d1ae1b..2c40b3bc4 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -33,7 +33,7 @@ type ExportCommand struct { // Name of the index & frame to export from. Index string Frame string - View string + // Filename to export to. Path string @@ -59,8 +59,6 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired - } else if !(cmd.View == pilosa.ViewStandard || cmd.View == pilosa.ViewInverse) { - return pilosa.ErrInvalidView } // Use output file, if specified. @@ -83,13 +81,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } // Determine slice count. - var maxSlices map[string]uint64 - if cmd.View == pilosa.ViewStandard { - maxSlices, err = client.MaxSliceByIndex(ctx) - } else if cmd.View == pilosa.ViewInverse { - maxSlices, err = client.MaxInverseSliceByIndex(ctx) - } - + maxSlices, err := client.MaxSliceByIndex(ctx) if err != nil { return errors.Wrap(err, "getting slice count") } @@ -97,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil { + if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, pilosa.ViewStandard, slice, w); err != nil { return errors.Wrap(err, "exporting") } } diff --git a/ctl/export_test.go b/ctl/export_test.go index 48a90a065..469ca7d75 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -41,13 +41,6 @@ func TestExportCommand_Validation(t *testing.T) { if err != pilosa.ErrFrameRequired { t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err) } - - cm.Frame = "f" - cm.View = "test" - err = cm.Run(context.Background()) - if err != pilosa.ErrInvalidView { - t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrInvalidView, err) - } } func TestExportCommand_Run(t *testing.T) { @@ -70,7 +63,6 @@ func TestExportCommand_Run(t *testing.T) { cm.Index = "i" cm.Frame = "f" - cm.View = pilosa.ViewStandard if err := cm.Run(context.Background()); err != nil { t.Fatalf("Export Run doesn't work: %s", err) } diff --git a/executor.go b/executor.go index 8a7c35d21..a86f2367e 100644 --- a/executor.go +++ b/executor.go @@ -80,36 +80,21 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Don't bother calculating slices for query types that don't require it. needsSlices := needsSlices(q.Calls) - // MaxSlice can differ between inverse and standard views, so we need - // to send queries to different slices based on orientation. - var inverseSlices []uint64 - - // If slices are specified, then use that value for slices or - // inverseSlices. If slices aren't specified, then include all of them. - if len(slices) > 0 { - // For inverse queries, the values of `slices` provided to the Execute() method - // on the remote node actually represents inverseSlices. - inverseSlices = slices - } else if needsSlices { + // If slices are specified, then use that value for slices. If slices aren't + // specified, then include all of them. + if needsSlices { // Round up the number of slices. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } maxSlice := idx.MaxSlice() - maxInverseSlice := idx.MaxInverseSlice() // Generate a slices of all slices. slices = make([]uint64, maxSlice+1) for i := range slices { slices[i] = uint64(i) } - - // Generate a slices of all inverse slices. - inverseSlices = make([]uint64, maxInverseSlice+1) - for i := range inverseSlices { - inverseSlices[i] = uint64(i) - } } // Optimize handling for bulk attribute insertion. @@ -120,23 +105,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { - if call.SupportsInverse() && needsSlices { - // Fetch frame & row label based on argument. - frame := call.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame - } - f := e.Holder.Frame(index, frame) - if f == nil { - return nil, ErrFrameNotFound - } - - // If this call is to an inverse frame send to a different list of slices. - if call.IsInverse(rowLabel, columnLabel) { - slices = inverseSlices - } - } - v, err := e.executeCall(ctx, index, call, slices, opt) if err != nil { return nil, err @@ -580,7 +548,6 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C // executeTopNSlice executes a TopN call for a single slice. func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { frame, _ := c.Args["frame"].(string) - inverse, _ := c.Args["inverse"].(bool) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -619,9 +586,6 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca // Determine view. view := ViewStandard - if inverse { - view = ViewInverse - } f := e.Holder.Fragment(index, frame, view, slice) if f == nil { @@ -687,30 +651,18 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. // Return an error if both the row and column label are specified. rowID, rowOK, rowErr := c.UintArg(rowLabel) - columnID, columnOK, columnErr := c.UintArg(columnLabel) - if rowErr != nil || columnErr != nil { - return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr) + if rowErr != nil { + return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr) } - if rowOK && columnOK { - return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel) - } else if !rowOK && !columnOK { - return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", rowLabel, columnLabel) + if !rowOK { + return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel) } - // Determine row or column orientation. - view, id := ViewStandard, rowID - if columnOK { - view, id = ViewInverse, columnID - if !f.InverseEnabled() { - return nil, fmt.Errorf("Bitmap() cannot retrieve columns unless inverse storage enabled") - } - } - - frag := e.Holder.Fragment(index, frame, view, slice) + frag := e.Holder.Fragment(index, frame, ViewStandard, slice) if frag == nil { return NewRow(), nil } - return frag.Row(id), nil + return frag.Row(rowID), nil } // executeIntersectSlice executes a intersect() call for a local slice. @@ -761,26 +713,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Read row & column id. - columnID, columnOK, err := c.UintArg(columnLabel) - if err != nil { - return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err) - } rowID, rowOK, err := c.UintArg(rowLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) } - - // Determine view. - var id uint64 - var viewName string - if columnOK && rowOK { - return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, rowLabel) - } else if !columnOK && !rowOK { - return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, rowLabel) - } else if columnOK { - viewName, id = ViewInverse, columnID - } else { - viewName, id = ViewStandard, rowID + if !rowOK { + return nil, fmt.Errorf("Range() must specify %q", rowLabel) } // Parse start time. @@ -811,12 +749,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Union bitmaps across all time-based subframes. row := &Row{} - for _, view := range ViewsByTimeRange(viewName, startTime, endTime, q) { + for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { f := e.Holder.Fragment(index, frame, view, slice) if f == nil { continue } - row = row.Union(f.Row(id)) + row = row.Union(f.Row(rowID)) } f.Stats.Count("range", 1, 1.0) return row, nil @@ -1033,7 +971,6 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, // executeClearBit executes a ClearBit() call. func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - view, _ := c.Args["view"].(string) frame, ok := c.Args["frame"].(string) if !ok { return false, errors.New("ClearBit() frame required") @@ -1065,30 +1002,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal } // Clear bits for each view. - switch view { - case ViewStandard: - return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt) - case ViewInverse: - return e.executeClearBitView(ctx, index, c, f, view, rowID, colID, opt) - case "": - var ret bool - if changed, err := e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt); err != nil { - return ret, err - } else if changed { - ret = true - } - - if f.InverseEnabled() { - if changed, err := e.executeClearBitView(ctx, index, c, f, ViewInverse, rowID, colID, opt); err != nil { - return ret, err - } else if changed { - ret = true - } - } - return ret, nil - default: - return false, fmt.Errorf("invalid view: %s", view) - } + return e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt) } // executeClearBitView executes a ClearBit() call for a single view. @@ -1123,7 +1037,6 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql // executeSetBit executes a SetBit() call. func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - view, _ := c.Args["view"].(string) frame, ok := c.Args["frame"].(string) if !ok { return false, errors.New("SetBit() field required: frame") @@ -1165,30 +1078,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, } // Set bits for each view. - switch view { - case ViewStandard: - return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt) - case ViewInverse: - return e.executeSetBitView(ctx, index, c, f, view, rowID, colID, timestamp, opt) - case "": - var ret bool - if changed, err := e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil { - return ret, err - } else if changed { - ret = true - } - - if f.InverseEnabled() { - if changed, err := e.executeSetBitView(ctx, index, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil { - return ret, err - } else if changed { - ret = true - } - } - return ret, nil - default: - return false, fmt.Errorf("invalid view: %s", view) - } + return e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt) } // executeSetBitView executes a SetBit() call for a specific view. diff --git a/executor_test.go b/executor_test.go index c6d2c6dda..0807228ac 100644 --- a/executor_test.go +++ b/executor_test.go @@ -33,7 +33,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) + f, err := index.CreateFrame("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } @@ -60,7 +60,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } - // Inhibit columns attributes. + // Inhibit column attributes. if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { t.Fatal(err) } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { @@ -83,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + if _, err := index.CreateFrame("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -100,14 +100,6 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } - - if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { - t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{10, 20}) { - t.Fatalf("unexpected columns: %+v", columns) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { - t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) - } }) } @@ -412,9 +404,9 @@ func TestExecutor_Execute_TopN(t *testing.T) { // Set columns for rows 0, 10, & 20 across two slices. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` SetBit(frame=f, row=0, col=0) @@ -431,7 +423,6 @@ func TestExecutor_Execute_TopN(t *testing.T) { } hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewInverse, 0).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() @@ -445,17 +436,6 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) - - t.Run("Inverse", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ - {ID: SliceWidth, Count: 3}, - {ID: 0, Count: 2}, - }) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) - } - }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { @@ -758,8 +738,7 @@ func TestExecutor_Execute_Range(t *testing.T) { // Create frame. if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - InverseEnabled: true, - TimeQuantum: pilosa.TimeQuantum("YMDH"), + TimeQuantum: pilosa.TimeQuantum("YMDH"), }); err != nil { t.Fatal(err) } @@ -788,14 +767,6 @@ func TestExecutor_Execute_Range(t *testing.T) { } }) - t.Run("Inverse", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { - t.Fatal(err) - } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 10}) { - t.Fatalf("unexpected columns: %+v", columns) - } - }) } // Ensure a Range(field) query can be executed. @@ -1246,7 +1217,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) + index.CreateFrame("f", pilosa.FrameOptions{}) targetAttrs := map[string]interface{}{ "foo": "bar", } diff --git a/frame.go b/frame.go index 52fa2e752..ab005edcc 100644 --- a/frame.go +++ b/frame.go @@ -31,8 +31,7 @@ import ( // Default frame settings. const ( - DefaultCacheType = CacheTypeRanked - DefaultInverseEnabled = false + DefaultCacheType = CacheTypeRanked // Default ranked frame cache DefaultCacheSize = 50000 @@ -54,11 +53,10 @@ type Frame struct { Stats StatsClient // Frame options. - inverseEnabled bool - cacheType string - cacheSize uint32 - timeQuantum TimeQuantum - fields []*Field + cacheType string + cacheSize uint32 + timeQuantum TimeQuantum + fields []*Field Logger Logger } @@ -82,9 +80,8 @@ func NewFrame(path, index, name string) (*Frame, error) { broadcaster: NopBroadcaster, Stats: NopStatsClient, - inverseEnabled: DefaultInverseEnabled, - cacheType: DefaultCacheType, - cacheSize: DefaultCacheSize, + cacheType: DefaultCacheType, + cacheSize: DefaultCacheSize, //timeQuantum //fields @@ -111,37 +108,18 @@ func (f *Frame) MaxSlice() uint64 { var max uint64 for _, view := range f.views { - if view.name == ViewInverse { - continue - } else if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max { + if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max { max = viewMaxSlice } } return max } -// MaxInverseSlice returns the max inverse slice in the frame. -func (f *Frame) MaxInverseSlice() uint64 { - f.mu.RLock() - defer f.mu.RUnlock() - - view := f.views[ViewInverse] - if view == nil { - return 0 - } - return view.MaxSlice() -} - // CacheType returns the caching mode for the frame. func (f *Frame) CacheType() string { return f.cacheType } -// InverseEnabled returns true if an inverse view is available. -func (f *Frame) InverseEnabled() bool { - return f.inverseEnabled -} - // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 func (f *Frame) SetCacheSize(v uint32) error { @@ -179,11 +157,10 @@ func (f *Frame) Options() FrameOptions { func (f *Frame) options() FrameOptions { return FrameOptions{ - InverseEnabled: f.inverseEnabled, - CacheType: f.cacheType, - CacheSize: f.cacheSize, - TimeQuantum: f.timeQuantum, - Fields: f.fields, + CacheType: f.cacheType, + CacheSize: f.cacheSize, + TimeQuantum: f.timeQuantum, + Fields: f.fields, } } @@ -255,7 +232,6 @@ func (f *Frame) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) if os.IsNotExist(err) { - f.inverseEnabled = DefaultInverseEnabled f.cacheType = DefaultCacheType f.cacheSize = DefaultCacheSize f.timeQuantum = "" @@ -270,7 +246,6 @@ func (f *Frame) loadMeta() error { } // Copy metadata fields. - f.inverseEnabled = pb.InverseEnabled f.cacheType = pb.CacheType if f.cacheType == "" { f.cacheType = DefaultCacheType @@ -532,11 +507,6 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { // createViewIfNotExistsBase returns the named view, creating it if necessary. // The returned bool indicates whether the view was created or not. func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { - // Don't create inverse views if they are not enabled. - if !f.InverseEnabled() && IsInverseView(name) { - return nil, false, ErrFrameInverseDisabled - } - f.mu.Lock() defer f.mu.Unlock() @@ -840,16 +810,14 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro timestamp = timestamps[i] } - var standard, inverse []string + var standard []string if timestamp == nil { standard = []string{ViewStandard} - inverse = []string{ViewInverse} } else { standard = ViewsByTime(ViewStandard, *timestamp, q) // In order to match the logic of `SetBit()`, we want bits // with timestamps to write to both time and standard views. standard = append(standard, ViewStandard) - inverse = ViewsByTime(ViewInverse, *timestamp, q) } // Attach bit to each standard view. @@ -860,34 +828,10 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro data.ColumnIDs = append(data.ColumnIDs, columnID) dataByFragment[key] = data } - - if f.inverseEnabled { - // Attach reversed bits to each inverse view. - for _, name := range inverse { - key := importKey{View: name, Slice: rowID / SliceWidth} - data := dataByFragment[key] - data.RowIDs = append(data.RowIDs, columnID) // reversed - data.ColumnIDs = append(data.ColumnIDs, rowID) // reversed - dataByFragment[key] = data - } - } } // Import into each fragment. for key, data := range dataByFragment { - // Skip inverse data if inverse is not enabled. - if !f.inverseEnabled && IsInverseView(key.View) { - continue - } - - // Re-sort data for inverse views. - if IsInverseView(key.View) { - sort.Sort(importBitSet{ - rowIDs: data.RowIDs, - columnIDs: data.ColumnIDs, - }) - } - view, err := f.CreateViewIfNotExists(key.View) if err != nil { return errors.Wrap(err, "creating view") @@ -1003,11 +947,10 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { - InverseEnabled bool `json:"inverseEnabled,omitempty"` - CacheType string `json:"cacheType,omitempty"` - CacheSize uint32 `json:"cacheSize,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - Fields []*Field `json:"fields,omitempty"` + CacheType string `json:"cacheType,omitempty"` + CacheSize uint32 `json:"cacheSize,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Fields []*Field `json:"fields,omitempty"` } // Encode converts o into its internal representation. @@ -1020,11 +963,10 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { return nil } return &internal.FrameMeta{ - InverseEnabled: o.InverseEnabled, - CacheType: o.CacheType, - CacheSize: o.CacheSize, - TimeQuantum: string(o.TimeQuantum), - Fields: encodeFields(o.Fields), + CacheType: o.CacheType, + CacheSize: o.CacheSize, + TimeQuantum: string(o.TimeQuantum), + Fields: encodeFields(o.Fields), } } @@ -1033,11 +975,10 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { return nil } return &FrameOptions{ - InverseEnabled: options.InverseEnabled, - CacheType: options.CacheType, - CacheSize: options.CacheSize, - TimeQuantum: TimeQuantum(options.TimeQuantum), - Fields: decodeFields(options.Fields), + CacheType: options.CacheType, + CacheSize: options.CacheSize, + TimeQuantum: TimeQuantum(options.TimeQuantum), + Fields: decodeFields(options.Fields), } } diff --git a/handler.go b/handler.go index e8d4b5ab4..10dd989d0 100644 --- a/handler.go +++ b/handler.go @@ -81,7 +81,7 @@ func NewHandler() *Handler { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") - h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse") + h.validators["GetSliceMax"] = queryValidationSpecRequired() h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice") h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") @@ -303,7 +303,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ Standard: h.API.MaxSlices(r.Context()), - Inverse: h.API.MaxInverseSlices(r.Context()), }); err != nil { h.Logger.Printf("write slices-max response error: %s", err) } @@ -311,7 +310,6 @@ func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { type getSlicesMaxResponse struct { Standard map[string]uint64 `json:"standard"` - Inverse map[string]uint64 `json:"inverse"` } // handleGetIndexes handles GET /index request. diff --git a/handler_internal_test.go b/handler_internal_test.go index 4603662e6..ceaeb707c 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -66,8 +66,8 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, - {json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}}, - {json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}}, + {json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"}, + {json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}}, {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { diff --git a/handler_test.go b/handler_test.go index 85bf152d6..51a0c604a 100644 --- a/handler_test.go +++ b/handler_test.go @@ -84,12 +84,10 @@ func TestHandler_Schema(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil { - t.Fatal(err) } if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { t.Fatal(err) @@ -107,8 +105,8 @@ func TestHandler_Schema(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -123,12 +121,10 @@ func TestHandler_Status(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil { - t.Fatal(err) } if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { t.Fatal(err) @@ -209,48 +205,7 @@ func TestHandler_MaxSlices(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can return the maxslice map for the inverse views. -func TestHandler_MaxSlices_Inverse(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true}) - if err != nil { - t.Fatal(err) - } - if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+1, nil); err != nil { - t.Fatal(err) - } else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+2, nil); err != nil { - t.Fatal(err) - } else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (3*SliceWidth)+4, nil); err != nil { - t.Fatal(err) - } - - f1, err := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true}) - if err != nil { - t.Fatal(err) - } - if _, err := f1.SetBit(pilosa.ViewStandard, 40, (0*SliceWidth)+1, nil); err != nil { - t.Fatal(err) - } else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+2, nil); err != nil { - t.Fatal(err) - } else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+4, nil); err != nil { - t.Fatal(err) - } - - h := test.NewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } diff --git a/holder.go b/holder.go index a80ba4539..6d13159b9 100644 --- a/holder.go +++ b/holder.go @@ -209,15 +209,6 @@ func (h *Holder) MaxSlices() map[string]uint64 { return a } -// MaxInverseSlices returns MaxInverseSlice map for all indexes. -func (h *Holder) MaxInverseSlices() map[string]uint64 { - a := make(map[string]uint64) - for _, index := range h.Indexes() { - a[index.Name()] = index.MaxInverseSlice() - } - return a -} - // Schema returns schema information for all indexes, frames, and views. func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo @@ -270,7 +261,6 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { func (h *Holder) EncodeMaxSlices() *internal.MaxSlices { return &internal.MaxSlices{ Standard: h.MaxSlices(), - Inverse: h.MaxInverseSlices(), } } diff --git a/index.go b/index.go index 0d8825234..534930cd5 100644 --- a/index.go +++ b/index.go @@ -38,8 +38,7 @@ type Index struct { frames map[string]*Frame // Max Slice on any node in the cluster, according to this node. - remoteMaxSlice uint64 - remoteMaxInverseSlice uint64 + remoteMaxSlice uint64 NewAttrStore func(string) AttrStore @@ -64,8 +63,7 @@ func NewIndex(path, name string) (*Index, error) { name: name, frames: make(map[string]*Frame), - remoteMaxSlice: 0, - remoteMaxInverseSlice: 0, + remoteMaxSlice: 0, NewAttrStore: NewNopAttrStore, columnAttrStore: NopAttrStore, @@ -236,30 +234,6 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) { i.remoteMaxSlice = newmax } -// MaxInverseSlice returns the max inverse slice in the index according to this node. -func (i *Index) MaxInverseSlice() uint64 { - if i == nil { - return 0 - } - i.mu.RLock() - defer i.mu.RUnlock() - - max := i.remoteMaxInverseSlice - for _, f := range i.frames { - if slice := f.MaxInverseSlice(); slice > max { - max = slice - } - } - return max -} - -// SetRemoteMaxInverseSlice sets the remote max inverse slice value received from another node. -func (i *Index) SetRemoteMaxInverseSlice(v uint64) { - i.mu.Lock() - defer i.mu.Unlock() - i.remoteMaxInverseSlice = v -} - // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } @@ -359,8 +333,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { f.cacheSize = opt.CacheSize } - f.inverseEnabled = opt.InverseEnabled - // Set fields. f.fields = opt.Fields diff --git a/index_test.go b/index_test.go index ea0747060..0a42804e2 100644 --- a/index_test.go +++ b/index_test.go @@ -98,48 +98,6 @@ func TestIndex_CreateFrame(t *testing.T) { } }) - t.Run("ErrInverseRangeAllowed", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - frame, err := index.CreateFrame("f", pilosa.FrameOptions{ - InverseEnabled: true, - Fields: []*pilosa.Field{ - &pilosa.Field{ - Name: "myfield", - Type: pilosa.FieldTypeInt, - Min: -20, - Max: 100, - }, - }, - }) - if err != nil { - t.Fatal(err) - } - - ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil) - if !ch || err != nil { - t.Fatal(ch, err) - } - ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil) - if !ch || err != nil { - t.Fatal(ch, err) - } - ch, err = frame.SetFieldValue(1, "myfield", 87) - if !ch || err != nil { - t.Fatal(ch, err) - } - views := frame.Views() - if len(views) != 3 { - var names string - for _, v := range views { - names = names + v.Name() + " " - } - t.Fatalf("Unexpected views: %s", names) - } - - }) - t.Run("ErrRangeCacheAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() diff --git a/internal/private.pb.go b/internal/private.pb.go index 4400677c3..35b452dce 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -70,11 +70,10 @@ func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } type FrameMeta struct { - InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` - CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` - CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` - TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"` + CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` + CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` + TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"` } func (m *FrameMeta) Reset() { *m = FrameMeta{} } @@ -82,13 +81,6 @@ func (m *FrameMeta) String() string { return proto.CompactTextString( func (*FrameMeta) ProtoMessage() {} func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } -func (m *FrameMeta) GetInverseEnabled() bool { - if m != nil { - return m.InverseEnabled - } - return false -} - func (m *FrameMeta) GetCacheType() string { if m != nil { return m.CacheType @@ -223,7 +215,6 @@ func (m *Cache) GetIDs() []uint64 { type MaxSlices struct { Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` - Inverse map[string]uint64 `protobuf:"bytes,2,rep,name=Inverse" json:"Inverse,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } func (m *MaxSlices) Reset() { *m = MaxSlices{} } @@ -238,17 +229,9 @@ func (m *MaxSlices) GetStandard() map[string]uint64 { return nil } -func (m *MaxSlices) GetInverse() map[string]uint64 { - if m != nil { - return m.Inverse - } - return nil -} - type CreateSliceMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` - IsInverse bool `protobuf:"varint,3,opt,name=IsInverse,proto3" json:"IsInverse,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` } func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} } @@ -270,13 +253,6 @@ func (m *CreateSliceMessage) GetSlice() uint64 { return 0 } -func (m *CreateSliceMessage) GetIsInverse() bool { - if m != nil { - return m.IsInverse - } - return false -} - type DeleteIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` } @@ -1059,16 +1035,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.InverseEnabled { - dAtA[i] = 0x10 - i++ - if m.InverseEnabled { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } if len(m.CacheType) > 0 { dAtA[i] = 0x1a i++ @@ -1289,22 +1255,6 @@ func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(v)) } } - if len(m.Inverse) > 0 { - for k, _ := range m.Inverse { - dAtA[i] = 0x12 - i++ - v := m.Inverse[k] - mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) - i = encodeVarintPrivate(dAtA, i, uint64(mapSize)) - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(k))) - i += copy(dAtA[i:], k) - dAtA[i] = 0x10 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(v)) - } - } return i, nil } @@ -1334,16 +1284,6 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) } - if m.IsInverse { - dAtA[i] = 0x18 - i++ - if m.IsInverse { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } return i, nil } @@ -2306,9 +2246,6 @@ func (m *IndexMeta) Size() (n int) { func (m *FrameMeta) Size() (n int) { var l int _ = l - if m.InverseEnabled { - n += 2 - } l = len(m.CacheType) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) @@ -2407,14 +2344,6 @@ func (m *MaxSlices) Size() (n int) { n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) } } - if len(m.Inverse) > 0 { - for k, v := range m.Inverse { - _ = k - _ = v - mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) - n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) - } - } return n } @@ -2428,9 +2357,6 @@ func (m *CreateSliceMessage) Size() (n int) { if m.Slice != 0 { n += 1 + sovPrivate(uint64(m.Slice)) } - if m.IsInverse { - n += 2 - } return n } @@ -2936,26 +2862,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.InverseEnabled = bool(v != 0) case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field CacheType", wireType) @@ -3761,113 +3667,6 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } m.Standard[mapkey] = mapvalue iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Inverse", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Inverse == nil { - m.Inverse = make(map[string]uint64) - } - var mapkey string - var mapvalue uint64 - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - } else { - iNdEx = entryPreIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > postIndex { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - m.Inverse[mapkey] = mapvalue - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -3966,26 +3765,6 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { break } } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field IsInverse", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.IsInverse = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7218,74 +6997,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1099 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xbd, 0x6b, 0x27, 0x7e, 0xae, 0x53, 0x67, 0x5a, 0xca, 0xb6, 0xaa, 0x82, 0x19, 0x15, - 0x6a, 0x38, 0x44, 0x25, 0xbd, 0x40, 0xa1, 0x52, 0x95, 0x38, 0x15, 0x8b, 0x48, 0x04, 0xe3, 0xa4, - 0x07, 0x24, 0x90, 0x26, 0xf6, 0x28, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0xf9, 0xd3, 0x03, 0x67, - 0x2e, 0xdc, 0x11, 0x1f, 0x85, 0x4f, 0xc0, 0x91, 0x8f, 0x80, 0xc2, 0x07, 0x01, 0xbd, 0x37, 0xb3, - 0x7f, 0x62, 0x3b, 0x4d, 0x09, 0xdc, 0xe6, 0xfd, 0x9d, 0xdf, 0xfb, 0x3b, 0xbb, 0xd0, 0x9e, 0xa4, - 0xe1, 0xb1, 0xd4, 0x6a, 0x7d, 0x92, 0x26, 0x3a, 0x61, 0xcb, 0x61, 0xac, 0x55, 0x1a, 0xcb, 0x88, - 0xb7, 0xa0, 0x19, 0xc4, 0x23, 0x75, 0xba, 0xa3, 0xb4, 0xe4, 0xbf, 0x39, 0xd0, 0x7c, 0x9e, 0xca, - 0xb1, 0x42, 0x8a, 0x7d, 0x00, 0x2b, 0x41, 0x7c, 0xac, 0xd2, 0x4c, 0x6d, 0xc7, 0xf2, 0x20, 0x52, - 0x23, 0xbf, 0xd6, 0x75, 0x7a, 0xcb, 0x62, 0x86, 0xcb, 0xee, 0x43, 0x73, 0x4b, 0x0e, 0x5f, 0xaa, - 0xbd, 0xb3, 0x89, 0xf2, 0xdd, 0xae, 0xd3, 0x6b, 0x8a, 0x92, 0x51, 0x48, 0x07, 0xe1, 0x2b, 0xe5, - 0x7b, 0x5d, 0xa7, 0xd7, 0x16, 0x25, 0x83, 0x75, 0xa1, 0xb5, 0x17, 0x8e, 0xd5, 0x37, 0x53, 0x19, - 0xeb, 0xe9, 0xd8, 0xaf, 0x93, 0x75, 0x95, 0xc5, 0x1e, 0x42, 0xe3, 0x79, 0xa8, 0xa2, 0x51, 0xe6, - 0x2f, 0x75, 0xdd, 0x5e, 0x6b, 0xe3, 0xe6, 0x7a, 0x8e, 0x7d, 0x9d, 0xf8, 0xc2, 0x8a, 0x39, 0x87, - 0x95, 0x60, 0x3c, 0x49, 0x52, 0x2d, 0x54, 0x36, 0x49, 0xe2, 0x4c, 0xb1, 0x0e, 0xb8, 0xdb, 0x69, - 0xea, 0x3b, 0xe4, 0x14, 0x8f, 0xfc, 0x47, 0xe8, 0x6c, 0x46, 0xc9, 0xf0, 0xa8, 0x2f, 0xb5, 0x14, - 0xea, 0x87, 0xa9, 0xca, 0x34, 0xbb, 0x0d, 0x75, 0xca, 0x80, 0xd5, 0x33, 0x04, 0x72, 0x29, 0x13, - 0x14, 0x73, 0x53, 0x18, 0x02, 0xb9, 0x64, 0x4f, 0x61, 0x7a, 0xc2, 0x10, 0xc8, 0x1d, 0x44, 0xe1, - 0xd0, 0x84, 0xe7, 0x09, 0x43, 0x30, 0x06, 0xde, 0x8b, 0x50, 0x9d, 0xd8, 0x98, 0xe8, 0xcc, 0x03, - 0x58, 0xad, 0xdc, 0x6f, 0x61, 0xde, 0x81, 0x86, 0x48, 0x4e, 0x82, 0x7e, 0xe6, 0x3b, 0x5d, 0xb7, - 0xe7, 0x09, 0x4b, 0x51, 0xe6, 0x92, 0x68, 0x3a, 0x8e, 0x51, 0x54, 0x23, 0x51, 0xc9, 0xe0, 0x77, - 0xa1, 0x4e, 0x69, 0xc4, 0x28, 0x4b, 0x5b, 0x3c, 0xf2, 0xbf, 0x1d, 0x68, 0xee, 0xc8, 0x53, 0x82, - 0x91, 0xb1, 0xa7, 0xb0, 0x3c, 0xd0, 0x32, 0x1e, 0xc9, 0x74, 0x44, 0x4a, 0xad, 0x8d, 0xf7, 0xca, - 0x14, 0x16, 0x6a, 0xeb, 0xb9, 0xce, 0x76, 0xac, 0xd3, 0x33, 0x51, 0x98, 0xb0, 0x27, 0xb0, 0x64, - 0xeb, 0x4d, 0x18, 0x5a, 0x1b, 0xdd, 0x45, 0xd6, 0x45, 0x4b, 0xa0, 0x71, 0x6e, 0x70, 0xef, 0x33, - 0x68, 0x5f, 0x70, 0x8b, 0x58, 0x8f, 0xd4, 0x59, 0x5e, 0x91, 0x23, 0x75, 0x86, 0xb9, 0x3b, 0x96, - 0xd1, 0xd4, 0xe4, 0xd9, 0x13, 0x86, 0x78, 0x52, 0xfb, 0xc4, 0xb9, 0xf7, 0x04, 0x6e, 0x54, 0xbd, - 0xfe, 0x1b, 0x5b, 0xfe, 0x3d, 0xb0, 0xad, 0x54, 0x49, 0xad, 0x08, 0xde, 0x8e, 0xca, 0x32, 0x79, - 0xa8, 0x2e, 0xaf, 0xb4, 0xa9, 0x5e, 0xad, 0x5a, 0xbd, 0xfb, 0xd0, 0x0c, 0xb2, 0x3c, 0x70, 0x97, - 0xfa, 0xbe, 0x64, 0xf0, 0x8f, 0x80, 0xf5, 0x55, 0xa4, 0xb4, 0xb2, 0xb3, 0xf3, 0x1a, 0xff, 0x7c, - 0x90, 0x63, 0xb9, 0x5a, 0x97, 0x3d, 0x04, 0x0f, 0x47, 0x8f, 0xa0, 0xb4, 0x36, 0x6e, 0x95, 0x99, - 0x2e, 0x66, 0x54, 0x90, 0x02, 0x0f, 0x73, 0xa7, 0x76, 0x5c, 0xaf, 0x08, 0x70, 0x41, 0x2b, 0xe7, - 0x57, 0xb9, 0xb3, 0x57, 0x15, 0x0b, 0xc0, 0x5e, 0xf5, 0x2c, 0x8f, 0xf5, 0xba, 0x57, 0xf1, 0xc3, - 0x02, 0x2c, 0x4e, 0xea, 0x75, 0xc0, 0xbe, 0x0f, 0x75, 0xb2, 0xb5, 0x68, 0xe7, 0x76, 0x80, 0x91, - 0xf2, 0x17, 0x05, 0xd4, 0xeb, 0x5e, 0x74, 0xbb, 0x7a, 0x51, 0x33, 0xf7, 0xfb, 0xad, 0xd5, 0xc5, - 0x99, 0xde, 0x45, 0x1b, 0xe3, 0x89, 0xce, 0x97, 0xd7, 0x6c, 0x26, 0x91, 0xe8, 0x1b, 0x97, 0x40, - 0xe6, 0xbb, 0x5d, 0x17, 0x7d, 0x13, 0xc1, 0x1f, 0x43, 0x63, 0x30, 0x7c, 0xa9, 0xc6, 0x92, 0x7d, - 0x88, 0x93, 0x36, 0x52, 0xa7, 0x2a, 0xb3, 0x73, 0x7a, 0x73, 0xa6, 0xfe, 0x22, 0x97, 0xf3, 0xbe, - 0x0d, 0xe9, 0x12, 0x40, 0x0d, 0xba, 0x3a, 0xf3, 0xbd, 0xb9, 0x8d, 0x89, 0x7c, 0x61, 0xc5, 0x7c, - 0x1b, 0xdc, 0x7d, 0x11, 0xe0, 0xfe, 0x21, 0x04, 0xb9, 0x17, 0x4b, 0xa1, 0xef, 0x2f, 0x92, 0x4c, - 0xdb, 0x04, 0xd1, 0x19, 0x79, 0x5f, 0x27, 0xa9, 0xa6, 0xf4, 0xb4, 0x05, 0x9d, 0xf9, 0x77, 0xe0, - 0xed, 0x26, 0x23, 0xc5, 0x56, 0xa0, 0x16, 0xf4, 0xad, 0x8f, 0x5a, 0xd0, 0x67, 0xef, 0x92, 0x7b, - 0x9b, 0x97, 0x76, 0x09, 0x62, 0x5f, 0x04, 0x82, 0x2e, 0x7e, 0x00, 0xed, 0x20, 0xdb, 0x4a, 0x92, - 0x74, 0x14, 0xc6, 0x52, 0x27, 0xa9, 0x9d, 0xb3, 0x8b, 0x4c, 0xfe, 0x0c, 0x3a, 0xe8, 0x7e, 0xa0, - 0xa5, 0x2e, 0xba, 0xef, 0x0e, 0x34, 0x90, 0x57, 0x5c, 0x67, 0x29, 0x9a, 0x65, 0xd4, 0xcb, 0x8b, - 0x4a, 0x04, 0xff, 0xca, 0x78, 0xd8, 0x3e, 0x56, 0xb1, 0xae, 0x34, 0x05, 0xd1, 0xe4, 0xa0, 0x2d, - 0x0c, 0xc1, 0xb8, 0x09, 0xc5, 0x62, 0x5e, 0x29, 0x31, 0x23, 0x57, 0x90, 0x8c, 0xff, 0xec, 0x00, - 0xe4, 0x80, 0xa6, 0x59, 0x61, 0xe2, 0x5c, 0x6e, 0xc2, 0x3e, 0xae, 0xec, 0xe3, 0xf9, 0x3e, 0x29, - 0x44, 0xa2, 0xb2, 0xb5, 0x7b, 0x79, 0x5b, 0xd8, 0x96, 0xef, 0x94, 0xfa, 0x86, 0x6f, 0xcb, 0x84, - 0xab, 0xa0, 0xbd, 0x15, 0x4d, 0x33, 0xad, 0x52, 0x8b, 0x08, 0xdf, 0x0d, 0xc3, 0x28, 0xf2, 0x53, - 0x32, 0x16, 0xa7, 0x88, 0x3d, 0x80, 0x3a, 0x22, 0x35, 0xbd, 0x39, 0x1f, 0x86, 0x11, 0xf2, 0x81, - 0x9d, 0x8e, 0x85, 0x6d, 0xc7, 0xc0, 0xa3, 0x2f, 0x00, 0xdb, 0x2e, 0xf4, 0xf8, 0x77, 0xc0, 0xdd, - 0x09, 0x63, 0x0a, 0xc1, 0x15, 0x78, 0x24, 0x8e, 0x3c, 0xa5, 0x97, 0x12, 0x39, 0x12, 0xf7, 0xe3, - 0xaa, 0xd9, 0x0e, 0x38, 0x0f, 0xd7, 0x99, 0xd9, 0xfc, 0xa1, 0x75, 0x2b, 0x0f, 0xed, 0x00, 0x56, - 0xcd, 0x26, 0xf8, 0x3f, 0x9d, 0xfe, 0x5a, 0x83, 0x55, 0xa1, 0xb2, 0xf0, 0x95, 0x0a, 0xe2, 0x4c, - 0xa7, 0xd3, 0xa1, 0x0e, 0x93, 0x18, 0xed, 0xbf, 0x4c, 0x0e, 0x6c, 0xaa, 0x5d, 0x61, 0x88, 0x37, - 0xe9, 0x24, 0xf6, 0x08, 0x5a, 0xb3, 0xdd, 0x3f, 0xaf, 0x5a, 0x55, 0x61, 0x8f, 0x60, 0x69, 0x90, - 0x4c, 0xd3, 0x61, 0x31, 0xdb, 0x77, 0x4a, 0x6d, 0x83, 0xcc, 0x88, 0x45, 0xae, 0x56, 0xe9, 0xa3, - 0xfa, 0xeb, 0xfb, 0x88, 0x3d, 0x9d, 0xe9, 0x23, 0xbf, 0x41, 0x06, 0xef, 0x94, 0x06, 0x17, 0xc4, - 0xe2, 0xa2, 0x36, 0xff, 0xc9, 0x81, 0x1b, 0x55, 0x08, 0x6f, 0x34, 0x18, 0x45, 0x45, 0x6a, 0x0b, - 0x2b, 0xe2, 0x2e, 0xaa, 0x88, 0x57, 0x56, 0xa4, 0x7c, 0xbb, 0xeb, 0x95, 0xb7, 0x9b, 0x1f, 0xc1, - 0xdd, 0xb9, 0x32, 0x6d, 0x25, 0xe3, 0x09, 0xf6, 0xc3, 0x7f, 0x28, 0x17, 0xae, 0x8c, 0x34, 0xb5, - 0x85, 0x6a, 0x0a, 0x43, 0xf0, 0x4f, 0xe1, 0xed, 0x81, 0xd2, 0x95, 0x22, 0xe5, 0xdd, 0xd6, 0x05, - 0x77, 0x57, 0x9d, 0x5c, 0x12, 0x3e, 0x8a, 0xf8, 0xe7, 0xe0, 0xef, 0x4f, 0x46, 0x52, 0xab, 0x6b, - 0x59, 0x6f, 0xc2, 0xf2, 0x5e, 0x32, 0x49, 0xa2, 0xe4, 0xf0, 0xec, 0x8a, 0x91, 0xf7, 0x61, 0xc9, - 0xec, 0x47, 0xf3, 0x19, 0xd9, 0x14, 0x39, 0xc9, 0x6f, 0x61, 0x43, 0x0f, 0x65, 0x34, 0x9c, 0x46, - 0x08, 0x03, 0xbf, 0x27, 0xb3, 0xcd, 0xce, 0xef, 0xe7, 0x6b, 0xce, 0x1f, 0xe7, 0x6b, 0xce, 0x9f, - 0xe7, 0x6b, 0xce, 0x2f, 0x7f, 0xad, 0xbd, 0x75, 0xd0, 0xa0, 0xbf, 0x86, 0xc7, 0xff, 0x04, 0x00, - 0x00, 0xff, 0xff, 0x60, 0xc2, 0x0f, 0x6a, 0x46, 0x0c, 0x00, 0x00, + // 1035 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1b, 0x45, + 0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x8c, 0x53, 0x67, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45, + 0x0d, 0x1c, 0xa2, 0x92, 0x5e, 0x78, 0x55, 0x8a, 0x12, 0xa7, 0x62, 0x11, 0x89, 0x60, 0x36, 0xe9, + 0x01, 0x89, 0xc3, 0xd4, 0x1e, 0xa5, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x1e, 0x3d, 0x70, 0x85, + 0x0b, 0x17, 0x4e, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f, + 0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xfb, 0xbd, 0x5f, 0xbf, 0xef, 0x9b, 0x85, 0xee, 0x24, 0x8d, 0xce, + 0xb9, 0x12, 0xdb, 0x93, 0x54, 0x2a, 0x49, 0x56, 0xa3, 0x44, 0x89, 0x34, 0xe1, 0x31, 0xed, 0x40, + 0x3b, 0x48, 0x46, 0xe2, 0xf2, 0x50, 0x28, 0x4e, 0x7f, 0x77, 0xa0, 0xfd, 0x34, 0xe5, 0x63, 0xa1, + 0x29, 0xf2, 0x3e, 0xb4, 0xf7, 0xf9, 0xf0, 0x85, 0x38, 0xbe, 0x9a, 0x08, 0xdf, 0xed, 0x3b, 0x5b, + 0x6d, 0x56, 0x31, 0x4a, 0x69, 0x18, 0xbd, 0x14, 0xbe, 0xd7, 0x77, 0xb6, 0xba, 0xac, 0x62, 0x90, + 0x3e, 0x74, 0x8e, 0xa3, 0xb1, 0xf8, 0x3e, 0xe7, 0x89, 0xca, 0xc7, 0x7e, 0x13, 0xad, 0xeb, 0x2c, + 0xf2, 0x10, 0x5a, 0x4f, 0x23, 0x11, 0x8f, 0x32, 0x7f, 0xa5, 0xef, 0x6e, 0x75, 0x76, 0xee, 0x6c, + 0x17, 0x39, 0x6d, 0x23, 0x9f, 0x59, 0x31, 0xa5, 0xb0, 0x16, 0x8c, 0x27, 0x32, 0x55, 0x4c, 0x64, + 0x13, 0x99, 0x64, 0x82, 0xf4, 0xc0, 0x3d, 0x48, 0x53, 0xdf, 0x41, 0xa7, 0xfa, 0x93, 0xfe, 0x0c, + 0xbd, 0xbd, 0x58, 0x0e, 0xcf, 0x06, 0x5c, 0x71, 0x26, 0x7e, 0xca, 0x45, 0xa6, 0xc8, 0x3d, 0x68, + 0x62, 0x65, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x15, 0xfa, 0x0d, 0xc3, 0x45, 0x42, 0x73, 0xd1, 0x1e, + 0xcb, 0xf4, 0x98, 0x21, 0x34, 0x37, 0x8c, 0xa3, 0xa1, 0x29, 0xcf, 0x63, 0x86, 0x20, 0x04, 0xbc, + 0x67, 0x91, 0xb8, 0xb0, 0x35, 0xe1, 0x37, 0x0d, 0x60, 0xbd, 0x16, 0xdf, 0xa6, 0xb9, 0x01, 0x2d, + 0x26, 0x2f, 0x82, 0x41, 0xe6, 0x3b, 0x7d, 0x77, 0xcb, 0x63, 0x96, 0xc2, 0xce, 0xc9, 0x38, 0x1f, + 0x27, 0x5a, 0xd4, 0x40, 0x51, 0xc5, 0xa0, 0xf7, 0xa1, 0x89, 0x6d, 0xd4, 0x55, 0x56, 0xb6, 0xfa, + 0x93, 0xfe, 0xe2, 0x40, 0xfb, 0x90, 0x5f, 0x62, 0x1a, 0x19, 0x79, 0x02, 0xab, 0xa1, 0xe2, 0xc9, + 0x88, 0xa7, 0x23, 0x54, 0xea, 0xec, 0x7c, 0x58, 0xb5, 0xb0, 0x54, 0xdb, 0x2e, 0x74, 0x0e, 0x12, + 0x95, 0x5e, 0xb1, 0xd2, 0xe4, 0xbd, 0x2f, 0xa1, 0x7b, 0x43, 0xa4, 0xe3, 0x9d, 0x89, 0xab, 0xa2, + 0xab, 0x67, 0xe2, 0x4a, 0xd7, 0x7f, 0xce, 0xe3, 0xdc, 0xf4, 0xca, 0x63, 0x86, 0xf8, 0xa2, 0xf1, + 0x99, 0x43, 0x77, 0x81, 0xec, 0xa7, 0x82, 0x2b, 0x81, 0x41, 0x0e, 0x45, 0x96, 0xf1, 0x53, 0xb1, + 0xb8, 0xe3, 0xa6, 0x8b, 0x8d, 0x5a, 0x17, 0xe9, 0x27, 0x40, 0x06, 0x22, 0x16, 0x4a, 0x58, 0xf4, + 0xbd, 0xc2, 0x03, 0x0d, 0x8b, 0x68, 0xb7, 0xeb, 0x92, 0x87, 0xe0, 0x69, 0xf0, 0x62, 0xb0, 0xce, + 0xce, 0xdd, 0xaa, 0x23, 0x25, 0xca, 0x19, 0x2a, 0xd0, 0xa8, 0x70, 0x6a, 0x01, 0x7f, 0x4b, 0x09, + 0x73, 0x40, 0x53, 0x84, 0x72, 0xa7, 0x43, 0x95, 0x2b, 0x64, 0x43, 0xed, 0x16, 0xb5, 0x2e, 0x1b, + 0x8a, 0x9e, 0x96, 0xc9, 0xea, 0x9d, 0x58, 0x26, 0xd9, 0x8f, 0xa0, 0x89, 0xb6, 0x36, 0xdb, 0x99, + 0x6d, 0x33, 0x52, 0xfa, 0xac, 0x4c, 0x75, 0xd9, 0x40, 0xf7, 0xea, 0x81, 0xda, 0x85, 0xdf, 0x1f, + 0xac, 0xae, 0xde, 0x9e, 0x23, 0x6d, 0x63, 0x3c, 0xe1, 0xf7, 0xe2, 0x99, 0x4d, 0x35, 0x52, 0xfb, + 0xd6, 0xeb, 0x96, 0xf9, 0x6e, 0xdf, 0xd5, 0xbe, 0x91, 0xa0, 0x8f, 0xa1, 0x15, 0x0e, 0x5f, 0x88, + 0x31, 0x27, 0x1f, 0xc3, 0x0a, 0xa6, 0x26, 0x32, 0xbb, 0x11, 0x77, 0xa6, 0xe6, 0xcf, 0x0a, 0x39, + 0x1d, 0xd8, 0x92, 0x16, 0x24, 0xd4, 0xc2, 0xd0, 0x99, 0xef, 0xcd, 0xdc, 0x26, 0xcd, 0x67, 0x56, + 0x4c, 0x0f, 0xc0, 0x3d, 0x61, 0x81, 0xde, 0x74, 0xcc, 0xa0, 0xf0, 0x62, 0x29, 0xed, 0xfb, 0x6b, + 0x99, 0x29, 0xdb, 0x20, 0xfc, 0xd6, 0xbc, 0xef, 0x64, 0xaa, 0xb0, 0x3d, 0x5d, 0x86, 0xdf, 0xf4, + 0x47, 0xf0, 0x8e, 0xe4, 0x48, 0x90, 0x35, 0x68, 0x04, 0x03, 0xeb, 0xa3, 0x11, 0x0c, 0xc8, 0x07, + 0xe8, 0xde, 0xf6, 0xa5, 0x5b, 0x25, 0x71, 0xc2, 0x02, 0x86, 0x81, 0x1f, 0x40, 0x37, 0xc8, 0xf6, + 0xa5, 0x4c, 0x47, 0x51, 0xc2, 0x95, 0x4c, 0xd1, 0xeb, 0x2a, 0xbb, 0xc9, 0xa4, 0xbb, 0xd0, 0xd3, + 0xee, 0x43, 0xc5, 0x55, 0x89, 0xbe, 0x0d, 0x68, 0x69, 0x5e, 0x19, 0xce, 0x52, 0xb8, 0xad, 0x5a, + 0xaf, 0x18, 0x2a, 0x12, 0xf4, 0x5b, 0xe3, 0xe1, 0xe0, 0x5c, 0x24, 0xaa, 0x06, 0x0a, 0xa4, 0xd1, + 0x41, 0x97, 0x19, 0x82, 0x50, 0x53, 0x8a, 0xcd, 0x79, 0xad, 0xca, 0x59, 0x73, 0x19, 0xca, 0xe8, + 0x6f, 0x0e, 0x40, 0x91, 0x50, 0x9e, 0x95, 0x26, 0xce, 0x62, 0x13, 0xf2, 0x69, 0xed, 0xf2, 0xcd, + 0xe2, 0xa4, 0x14, 0xb1, 0xda, 0x7d, 0xdc, 0x2a, 0x60, 0x61, 0x21, 0xdf, 0xab, 0xf4, 0x0d, 0xdf, + 0x8e, 0x49, 0x9f, 0x82, 0xee, 0x7e, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0xa1, 0x0d, 0xa3, + 0xec, 0x4f, 0xc5, 0x98, 0xdf, 0x22, 0xf2, 0x00, 0x9a, 0x3a, 0x53, 0x83, 0xcd, 0xd9, 0x32, 0x8c, + 0x90, 0x86, 0x76, 0x3b, 0xe6, 0xc2, 0x8e, 0x80, 0x87, 0x6f, 0xad, 0x85, 0x0b, 0x3e, 0xb3, 0x3d, + 0x70, 0x0f, 0xa3, 0x04, 0x4b, 0x70, 0x99, 0xfe, 0x44, 0x0e, 0xbf, 0xc4, 0x37, 0x49, 0x73, 0xb8, + 0xbe, 0x8f, 0xeb, 0xe6, 0x3a, 0xe8, 0x7d, 0x58, 0x66, 0x67, 0x8b, 0x27, 0xcd, 0xad, 0x3d, 0x69, + 0x21, 0xac, 0x9b, 0x4b, 0xf0, 0x26, 0x9d, 0xfe, 0xd9, 0x80, 0x75, 0x26, 0xb2, 0xe8, 0xa5, 0x08, + 0x92, 0x4c, 0xa5, 0xf9, 0x50, 0x45, 0x32, 0xd1, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xd5, 0x2e, 0x33, + 0xc4, 0xeb, 0x20, 0x89, 0x3c, 0x82, 0xce, 0x34, 0xfa, 0x67, 0x55, 0xeb, 0x2a, 0xe4, 0x11, 0xac, + 0x84, 0x32, 0x4f, 0x87, 0xe5, 0x6e, 0x6f, 0x54, 0xda, 0x26, 0x33, 0x23, 0x66, 0x85, 0x5a, 0x0d, + 0x47, 0xcd, 0x57, 0xe3, 0x88, 0x3c, 0x99, 0xc2, 0x91, 0xdf, 0x42, 0x83, 0x77, 0x2b, 0x83, 0x1b, + 0x62, 0x76, 0x53, 0x9b, 0xfe, 0xea, 0xc0, 0xdb, 0xf5, 0x14, 0x5e, 0x6b, 0x31, 0xca, 0x89, 0x34, + 0xe6, 0x4e, 0xc4, 0x9d, 0x37, 0x11, 0xaf, 0x9a, 0x48, 0xf5, 0x3a, 0x37, 0xeb, 0xaf, 0xf3, 0x19, + 0xdc, 0x9f, 0x19, 0xd3, 0xbe, 0x1c, 0x4f, 0x34, 0x1e, 0xfe, 0xc7, 0xb8, 0xf4, 0xc9, 0x48, 0x53, + 0x3b, 0xa8, 0x36, 0x33, 0x04, 0xfd, 0x1c, 0xde, 0x09, 0x85, 0xaa, 0x0d, 0xa9, 0x40, 0x5b, 0x1f, + 0xdc, 0x23, 0x71, 0xb1, 0xa0, 0x7c, 0x2d, 0xa2, 0x5f, 0x81, 0x7f, 0x32, 0x19, 0x71, 0x25, 0x96, + 0xb2, 0xde, 0x83, 0xd5, 0x63, 0x39, 0x91, 0xb1, 0x3c, 0xbd, 0xba, 0x65, 0xe5, 0x7d, 0x58, 0x31, + 0xf7, 0xd1, 0xfc, 0xb0, 0xb5, 0x59, 0x41, 0xd2, 0xbb, 0x1a, 0xd0, 0x43, 0x1e, 0x0f, 0xf3, 0x58, + 0xa7, 0xa1, 0xff, 0xdc, 0xb2, 0xbd, 0xde, 0x5f, 0xd7, 0x9b, 0xce, 0xdf, 0xd7, 0x9b, 0xce, 0x3f, + 0xd7, 0x9b, 0xce, 0x1f, 0xff, 0x6e, 0xbe, 0xf5, 0xbc, 0x85, 0xff, 0xdd, 0x8f, 0xff, 0x0b, 0x00, + 0x00, 0xff, 0xff, 0xd3, 0x15, 0x68, 0xea, 0x88, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index e1642698d..52e587f4b 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -6,7 +6,6 @@ message IndexMeta { } message FrameMeta { - bool InverseEnabled = 2; string CacheType = 3; uint32 CacheSize = 4; string TimeQuantum = 5; @@ -36,13 +35,11 @@ message Cache { message MaxSlices { map Standard = 1; - map Inverse = 2; } message CreateSliceMessage { string Index = 1; uint64 Slice = 2; - bool IsInverse = 3; } message DeleteIndexMessage { diff --git a/pilosa.go b/pilosa.go index bad8badc3..7ddea28eb 100644 --- a/pilosa.go +++ b/pilosa.go @@ -32,10 +32,9 @@ var ( ErrIndexNotFound = errors.New("index not found") // ErrFrameRequired is returned when no frame is specified. - ErrFrameRequired = errors.New("frame required") - ErrFrameExists = errors.New("frame already exists") - ErrFrameNotFound = errors.New("frame not found") - ErrFrameInverseDisabled = errors.New("frame inverse disabled") + ErrFrameRequired = errors.New("frame required") + ErrFrameExists = errors.New("frame already exists") + ErrFrameNotFound = errors.New("frame not found") ErrFieldNotFound = errors.New("field not found") ErrFieldExists = errors.New("field already exists") diff --git a/pql/ast.go b/pql/ast.go index e26432ad3..c3deff9dd 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -177,34 +177,6 @@ func (c *Call) String() string { return buf.String() } -// SupportsInverse indicates that the call may be on an inverse frame. -func (c *Call) SupportsInverse() bool { - return c.Name == "Bitmap" || c.Name == "TopN" -} - -// IsInverse specifies if the call is for an inverse view. -// Return defaults to false unless absolutely sure of inversion. -func (c *Call) IsInverse(rowLabel, columnLabel string) bool { - if c.SupportsInverse() { - // Top-n has an explicit inverse flag. - if c.Name == "TopN" { - inverse, _ := c.Args["inverse"].(bool) - return inverse - } - - // Bitmap calls use the row/column labels to determine whether inverse. - _, rowOK, rowErr := c.UintArg(rowLabel) - _, columnOK, columnErr := c.UintArg(columnLabel) - if rowErr != nil || columnErr != nil { - return false - } - if !rowOK && columnOK { - return true - } - } - return false -} - // HasConditionArg returns true if any arg is a conditional. func (c *Call) HasConditionArg() bool { for _, v := range c.Args { diff --git a/pql/ast_test.go b/pql/ast_test.go index 9dd3b693f..f5d75e9de 100644 --- a/pql/ast_test.go +++ b/pql/ast_test.go @@ -67,69 +67,3 @@ func TestCondition_Value(t *testing.T) { } }) } - -// Ensure call can be converted into a string. -func TestCall_SupportsInverse(t *testing.T) { - t.Run("Bitmap", func(t *testing.T) { - q, err := pql.ParseString(`Bitmap()`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].SupportsInverse() != true { - t.Fatalf("call should support inverse: %s", q.Calls[0]) - } - }) - t.Run("Count Bitmap", func(t *testing.T) { - q, err := pql.ParseString(`Count(Bitmap())`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].SupportsInverse() == true { - t.Fatalf("call should not support inverse: %s", q.Calls[0]) - } - }) - t.Run("Union Bitmaps", func(t *testing.T) { - q, err := pql.ParseString(`Union(Bitmap(), Bitmap())`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].SupportsInverse() == true { - t.Fatalf("call should not support inverse: %s", q.Calls[0]) - } - }) - -} - -// Ensure call is correctly determined to be against an inverse view. -func TestCall_IsInverse(t *testing.T) { - t.Run("Bitmap Row", func(t *testing.T) { - q, err := pql.ParseString(`Bitmap(frame="f", row=1)`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].IsInverse("row", "col") != false { - t.Fatalf("incorrect call inverse: %s", q.Calls[0]) - } - }) - t.Run("Bitmap Column", func(t *testing.T) { - q, err := pql.ParseString(`Bitmap(frame="f", col=1)`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].IsInverse("row", "col") != true { - t.Fatalf("incorrect call inverse: %s", q.Calls[0]) - } - }) - t.Run("Bitmap Column No Label", func(t *testing.T) { - q, err := pql.ParseString(`Bitmap(frame="f", col=1)`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].IsInverse("rowX", "colX") != false { - t.Fatalf("incorrect call inverse: %s", q.Calls[0]) - } - }) - t.Run("Count", func(t *testing.T) { - q, err := pql.ParseString(`Count(Bitmap(frame="f", col=1))`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].IsInverse("row", "col") != false { - t.Fatalf("incorrect call inverse: %s", q.Calls[0]) - } - }) - -} diff --git a/server.go b/server.go index 2731ad71a..b4e75bdbf 100644 --- a/server.go +++ b/server.go @@ -440,11 +440,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error { if idx == nil { return fmt.Errorf("Local Index not found: %s", obj.Index) } - if obj.IsInverse { - idx.SetRemoteMaxInverseSlice(obj.Slice) - } else { - idx.SetRemoteMaxSlice(obj.Slice) - } + idx.SetRemoteMaxSlice(obj.Slice) case *internal.CreateIndexMessage: opt := IndexOptions{} _, err := s.Holder.CreateIndex(obj.Index, opt) @@ -569,7 +565,7 @@ func (s *Server) SendTo(to *Node, pb proto.Message) error { // where a node fails to receive a Broadcast message, or // when a new (empty) node needs to get in sync with the // rest of the cluster, two things are shared via gossip: -// - MaxSlice/MaxInverseSlice by Index +// - MaxSlice by Index // - Schema // In a gossip implementation, memberlist.Delegate.LocalState() uses this. func (s *Server) LocalStatus() (proto.Message, error) { @@ -625,7 +621,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { return errors.Wrap(err, "applying schema") } - // Sync maxSlices (standard). + // Sync maxSlices. oldmaxslices := s.Holder.MaxSlices() for index, newMax := range ns.MaxSlices.Standard { localIndex := s.Holder.Index(index) @@ -641,22 +637,6 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error { } } - // Sync maxSlices (inverse). - oldMaxInverseSlices := s.Holder.MaxInverseSlices() - for index, newMaxInverse := range ns.MaxSlices.Inverse { - localIndex := s.Holder.Index(index) - // if we don't know about an index locally, log an error because - // indexes should be created and synced prior to slice creation - if localIndex == nil { - s.logger.Printf("Local Index not found: %s", index) - continue - } - if newMaxInverse > oldMaxInverseSlices[index] { - oldMaxInverseSlices[index] = newMaxInverse - localIndex.SetRemoteMaxInverseSlice(newMaxInverse) - } - } - return nil } diff --git a/server/server_test.go b/server/server_test.go index c7154daee..b70b8a046 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -26,7 +26,6 @@ import ( "strings" "testing" "testing/quick" - "time" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" @@ -69,8 +68,8 @@ func TestMain_Set_Quick(t *testing.T) { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "columns": columnIDs, - "attrs": map[string]interface{}{}, + "columns": columnIDs, + "attrs": map[string]interface{}{}, }, }, }) + "\n" @@ -92,8 +91,8 @@ func TestMain_Set_Quick(t *testing.T) { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "columns": columnIDs, - "attrs": map[string]interface{}{}, + "columns": columnIDs, + "attrs": map[string]interface{}{}, }, }, }) + "\n" @@ -236,49 +235,6 @@ func TestMain_SetColumnAttrs(t *testing.T) { } } -// Ensure inverse slices get handled correctly in a multi-node query. -func TestMain_InverseSlices(t *testing.T) { - mains := test.MustRunMainWithCluster(t, 2) - - m0 := mains[0] - m1 := mains[1] - - // Make sure to use node0 in the cluster. - var m *test.Main - if m0.Server.NodeID < m1.Server.NodeID { - m = m0 - } else { - m = m1 - } - - // Create frames. - client := m.Client() - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal("create index:", err) - } - if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { - t.Fatal("create frame:", err) - } - - // Write data on cluster. - if _, err := m.Query("i", "", fmt.Sprintf(` - SetBit(col=1, frame="f", row=1000) - SetBit(col=1, frame="f", row=2000) - SetBit(col=1, frame="f", row=%d) - `, 1*pilosa.SliceWidth)); err != nil { - t.Fatal("setting columns:", err) - } - - time.Sleep(1 * time.Second) - - // Query the cluster. - if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil { - t.Fatal("another bitmap query:", err) - } else if res != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" { - t.Fatalf("unexpected result: %s", res) - } -} - // Ensure program can set columns on one cluster and then restore to a second cluster. func TestMain_FrameRestore(t *testing.T) { mains1 := test.MustRunMainWithCluster(t, 2) diff --git a/view.go b/view.go index e825aa8ef..f90bf64e9 100644 --- a/view.go +++ b/view.go @@ -30,14 +30,13 @@ import ( // View layout modes. const ( ViewStandard = "standard" - ViewInverse = "inverse" ViewFieldPrefix = "field_" ) // IsValidView returns true if name is valid. func IsValidView(name string) bool { - return name == ViewStandard || name == ViewInverse + return name == ViewStandard } // View represents a container for frame data. @@ -252,9 +251,8 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { // Send the create slice message to all nodes. err := v.broadcaster.SendAsync( &internal.CreateSliceMessage{ - Index: v.index, - Slice: slice, - IsInverse: IsInverseView(v.name), + Index: v.index, + Slice: slice, }) if err != nil { return nil, errors.Wrap(err, "sending message") @@ -428,11 +426,6 @@ func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint6 return r, nil } -// IsInverseView returns true if the view is used for storing an inverted representation. -func IsInverseView(name string) bool { - return strings.HasPrefix(name, ViewInverse) -} - // ViewInfo represents schema information for a view. type ViewInfo struct { Name string `json:"name"` diff --git a/webui/assets/main.js b/webui/assets/main.js index b5ff2ab3c..265f51997 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -581,14 +581,11 @@ function parse_query(query, indexname) { function parse_options(option_str) { var int_keys = ["cacheSize"]; - var bool_keys = ["inverseEnabled"]; var options = {}; for (var i = 0; i < option_str.length; i++) { var parts = option_str[i].split('='); if (int_keys.indexOf(parts[0]) !== -1 ){ options[parts[0]] = Number(parts[1]) - } else if (bool_keys.indexOf(parts[0]) !== -1){ - options[parts[0]] = (parts[1] == "true") } else { options[parts[0]] = parts[1] } From b3b29fad47d4434b8d62669d3aa1ec6dbe9838e4 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 25 May 2018 17:12:56 -0500 Subject: [PATCH 12/13] correct bug where slices were being ignored --- executor.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/executor.go b/executor.go index a86f2367e..96f36dab5 100644 --- a/executor.go +++ b/executor.go @@ -82,7 +82,7 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // If slices are specified, then use that value for slices. If slices aren't // specified, then include all of them. - if needsSlices { + if len(slices) == 0 && needsSlices { // Round up the number of slices. idx := e.Holder.Index(index) if idx == nil { From a8795a7445956556da371124eaf1c62f3032c2f5 Mon Sep 17 00:00:00 2001 From: Matt Jaffee Date: Fri, 25 May 2018 19:30:48 -0500 Subject: [PATCH 13/13] remove comments about views no longer applicable --- executor.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/executor.go b/executor.go index 96f36dab5..59cd385a6 100644 --- a/executor.go +++ b/executor.go @@ -649,7 +649,6 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. return nil, ErrFrameNotFound } - // Return an error if both the row and column label are specified. rowID, rowOK, rowErr := c.UintArg(rowLabel) if rowErr != nil { return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr) @@ -1001,7 +1000,6 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) } - // Clear bits for each view. return e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt) } @@ -1077,7 +1075,6 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, timestamp = &t } - // Set bits for each view. return e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt) }