scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Bit -> Column migration

Browse source
This commit is contained in:
Todd Gruben 2018-05-23 15:05:22 -05:00
parent 3943e3e3cb
commit fb7bf11825
32 changed files with 710 additions and 710 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
api.go
View file

@ -101,7 +101,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
execOpts := &ExecOptions{
Remote: req.Remote,
ExcludeAttrs: req.ExcludeAttrs,
ExcludeBits: req.ExcludeBits,
ExcludeColumns: req.ExcludeColumns,
}
results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts)
if err != nil {
@ -110,7 +110,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
resp.Results = results
// Fill column attributes if requested.
if req.ColumnAttrs && !req.ExcludeBits {
if req.ColumnAttrs && !req.ExcludeColumns {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
@ -118,7 +118,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
if !ok {
continue
}
columnIDs = uint64Slice(columnIDs).merge(bm.Bits())
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
}
// Retrieve column attributes across all calls.
@ -305,7 +305,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin
// Wrap writer with a CSV writer.
cw := csv.NewWriter(w)
// Iterate over each bit.
// Iterate over each column.
if err := f.ForEachBit(func(rowID, columnID uint64) error {
return cw.Write([]string{
strconv.FormatUint(rowID, 10),
@ -784,7 +784,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
// Import into fragment.
err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -803,7 +803,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
// Import into fragment.
err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}

4
cache.go
View file

@ -168,7 +168,7 @@ func NewRankCache(maxEntries uint32) *RankCache {
func (c *RankCache) Add(id uint64, n uint64) {
c.mu.Lock()
defer c.mu.Unlock()
// Ignore if the bit count is below the threshold.
// Ignore if the column count is below the threshold.
if n < c.thresholdValue {
return
}
@ -469,7 +469,7 @@ type BitmapCache interface {
// SimpleCache implements BitmapCache
// it is meant to be a short-lived cache for cases where writes are continuing to access
// the same bit within a short time frame (i.e. good for write-heavy loads)
// the same column within a short time frame (i.e. good for write-heavy loads)
// A read-heavy use case would cause the cache to get bigger, potentially causing the
// node to run out of memory.
type SimpleCache struct {

86
client.go
View file

@ -279,15 +279,15 @@ func (c *InternalHTTPClient) QueryNode(ctx context.Context, uri *URI, index stri
return qresp, nil
}
// Import bulk imports bits for a single slice to a host.
func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error {
// Import bulk imports columns for a single slice to a host.
func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, slice uint64, columns []Bit) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
buf, err := marshalImportPayload(index, frame, slice, bits)
buf, err := marshalImportPayload(index, frame, slice, columns)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -308,15 +308,15 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl
return nil
}
// ImportK bulk imports bits to a host.
func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, bits []Bit) error {
// ImportK bulk imports columns to a host.
func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
buf, err := marshalImportPayloadK(index, frame, bits)
buf, err := marshalImportPayloadK(index, frame, columns)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -350,13 +350,13 @@ func (c *InternalHTTPClient) EnsureFrame(ctx context.Context, indexName string,
}
// marshalImportPayload marshalls the import parameters into a protobuf byte slice.
func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte, error) {
func marshalImportPayload(index, frame string, slice uint64, columns []Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowIDs := Bits(bits).RowIDs()
columnIDs := Bits(bits).ColumnIDs()
timestamps := Bits(bits).Timestamps()
rowIDs := Columns(columns).RowIDs()
columnIDs := Columns(columns).ColumnIDs()
timestamps := Columns(columns).Timestamps()
// Marshal bits to protobufs.
// Marshal columns to protobufs.
buf, err := proto.Marshal(&internal.ImportRequest{
Index: index,
Frame: frame,
@ -372,13 +372,13 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte
}
// marshalImportPayloadK marshalls the import parameters into a protobuf byte slice.
func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) {
func marshalImportPayloadK(index, frame string, columns []Bit) ([]byte, error) {
// Separate row and column IDs to reduce allocations.
rowKeys := Bits(bits).RowKeys()
columnKeys := Bits(bits).ColumnKeys()
timestamps := Bits(bits).Timestamps()
rowKeys := Columns(columns).RowKeys()
columnKeys := Columns(columns).ColumnKeys()
timestamps := Columns(columns).Timestamps()
// Marshal bits to protobufs.
// Marshal columns to protobufs.
buf, err := proto.Marshal(&internal.ImportRequest{
Index: index,
Frame: frame,
@ -465,7 +465,7 @@ func marshalImportValuePayload(index, frame, field string, slice uint64, vals []
columnIDs := FieldValues(vals).ColumnIDs()
values := FieldValues(vals).Values()
// Marshal bits to protobufs.
// Marshal columns to protobufs.
buf, err := proto.Marshal(&internal.ImportValueRequest{
Index: index,
Frame: frame,
@ -1140,7 +1140,7 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto
return nil
}
// Bit represents the location of a single bit.
// Bit represents the location of a single column.
type Bit struct {
RowID uint64
ColumnID uint64
@ -1149,13 +1149,13 @@ type Bit struct {
Timestamp int64
}
// Bits represents a slice of bits.
type Bits []Bit
// Columns represents a slice of columns.
type Columns []Bit
func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p Bits) Len() int { return len(p) }
func (p Columns) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p Columns) Len() int { return len(p) }
func (p Bits) Less(i, j int) bool {
func (p Columns) Less(i, j int) bool {
if p[i].RowID == p[j].RowID {
if p[i].ColumnID < p[j].ColumnID {
return p[i].Timestamp < p[j].Timestamp
@ -1166,7 +1166,7 @@ func (p Bits) Less(i, j int) bool {
}
// RowIDs returns a slice of all the row IDs.
func (p Bits) RowIDs() []uint64 {
func (p Columns) RowIDs() []uint64 {
other := make([]uint64, len(p))
for i := range p {
other[i] = p[i].RowID
@ -1175,7 +1175,7 @@ func (p Bits) RowIDs() []uint64 {
}
// ColumnIDs returns a slice of all the column IDs.
func (p Bits) ColumnIDs() []uint64 {
func (p Columns) ColumnIDs() []uint64 {
other := make([]uint64, len(p))
for i := range p {
other[i] = p[i].ColumnID
@ -1184,7 +1184,7 @@ func (p Bits) ColumnIDs() []uint64 {
}
// RowKeys returns a slice of all the row keys.
func (p Bits) RowKeys() []string {
func (p Columns) RowKeys() []string {
other := make([]string, len(p))
for i := range p {
other[i] = p[i].RowKey
@ -1193,7 +1193,7 @@ func (p Bits) RowKeys() []string {
}
// ColumnKeys returns a slice of all the column keys.
func (p Bits) ColumnKeys() []string {
func (p Columns) ColumnKeys() []string {
other := make([]string, len(p))
for i := range p {
other[i] = p[i].ColumnKey
@ -1202,7 +1202,7 @@ func (p Bits) ColumnKeys() []string {
}
// Timestamps returns a slice of all the timestamps.
func (p Bits) Timestamps() []int64 {
func (p Columns) Timestamps() []int64 {
other := make([]int64, len(p))
for i := range p {
other[i] = p[i].Timestamp
@ -1210,17 +1210,17 @@ func (p Bits) Timestamps() []int64 {
return other
}
// GroupBySlice returns a map of bits by slice.
func (p Bits) GroupBySlice() map[uint64][]Bit {
// GroupBySlice returns a map of columns by slice.
func (p Columns) GroupBySlice() map[uint64][]Bit {
m := make(map[uint64][]Bit)
for _, bit := range p {
slice := bit.ColumnID / SliceWidth
m[slice] = append(m[slice], bit)
for _, column := range p {
slice := column.ColumnID / SliceWidth
m[slice] = append(m[slice], column)
}
for slice, bits := range m {
sort.Sort(Bits(bits))
m[slice] = bits
for slice, columns := range m {
sort.Sort(Columns(columns))
m[slice] = columns
}
return m
@ -1277,12 +1277,12 @@ func (p FieldValues) GroupBySlice() map[uint64][]FieldValue {
return m
}
// BitsByPos represents a slice of bits sorted by internal position.
type BitsByPos []Bit
// ColumnsByPos represents a slice of columns sorted by internal position.
type ColumnsByPos []Bit
func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p BitsByPos) Len() int { return len(p) }
func (p BitsByPos) Less(i, j int) bool {
func (p ColumnsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p ColumnsByPos) Len() int { return len(p) }
func (p ColumnsByPos) Less(i, j int) bool {
p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID)
if p0 == p1 {
return p[i].Timestamp < p[j].Timestamp
@ -1320,8 +1320,8 @@ type InternalClient interface {
FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error)
Query(ctx context.Context, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
QueryNode(ctx context.Context, uri *URI, index string, queryRequest *internal.QueryRequest) (*internal.QueryResponse, error)
Import(ctx context.Context, index, frame string, slice uint64, bits []Bit) error
ImportK(ctx context.Context, index, frame string, bits []Bit) error
Import(ctx context.Context, index, frame string, slice uint64, columns []Bit) error
ImportK(ctx context.Context, index, frame string, columns []Bit) error
EnsureIndex(ctx context.Context, name string, options IndexOptions) error
EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error
ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error

90
client_test.go
View file

@ -110,26 +110,26 @@ func TestClient_MultiNode(t *testing.T) {
}
}
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(100, baseBit0+10)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetBits(22, baseBit0+1, baseBit0+2, baseBit0+10)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(100, baseBit0+10)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(4, baseBit0+10, baseBit0+11, baseBit0+12)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(4, baseBit0+10, baseBit0+11, baseBit0+12, baseBit0+13, baseBit0+14, baseBit0+15)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(2, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(3, baseBit0+1, baseBit0+2, baseBit0+3, baseBit0+4, baseBit0+5)
hldr[0].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[0]).MustSetColumns(22, baseBit0+1, baseBit0+2, baseBit0+10)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(1, baseBit1+4)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetBits(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(99, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(100, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6, baseBit1+7, baseBit1+8, baseBit1+9, baseBit1+10)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(98, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5, baseBit1+6)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(1, baseBit1+4)
hldr[1].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[1]).MustSetColumns(22, baseBit1+1, baseBit1+2, baseBit1+3, baseBit1+4, baseBit1+5)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(21, baseBit2+10)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(100, baseBit2+10)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(99, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(98, baseBit2+10, baseBit2+11)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetBits(22, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(24, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13, baseBit2+14)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(20, baseBit2+10, baseBit2+11, baseBit2+12, baseBit2+13)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(21, baseBit2+10)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(100, baseBit2+10)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(99, baseBit2+10, baseBit2+11, baseBit2+12)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(98, baseBit2+10, baseBit2+11)
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).MustSetColumns(22, baseBit2+10, baseBit2+11, baseBit2+12)
// Rebuild the RankCache.
// We have to do this to avoid the 10-second cache invalidation delay
@ -232,11 +232,11 @@ func TestClient_Import(t *testing.T) {
}
// Verify data.
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
@ -283,14 +283,14 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
}
// Verify data.
if a := f.Row(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f.Row(1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := f.Row(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f.Row(5).Columns(); !reflect.DeepEqual(a, []uint64{0, 200}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := f.Row(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f.Row(6).Columns(); !reflect.DeepEqual(a, []uint64{200}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
@ -375,10 +375,10 @@ func TestClient_BackupRestore(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(100, 1, 2, 3, SliceWidth-1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(100, SliceWidth, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetColumns(100, (5*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(200, 20000)
s := test.NewServer()
defer s.Close()
@ -403,17 +403,17 @@ func TestClient_BackupRestore(t *testing.T) {
}
// Verify data.
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
t.Fatalf("unexpected bits(0): %+v", a)
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) {
t.Fatalf("unexpected bits: %+v", a)
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Columns(); !reflect.DeepEqual(a, []uint64{20000}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
@ -468,8 +468,8 @@ func TestClient_BackupInverseView(t *testing.T) {
}
// Verify data.
if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
}
@ -479,7 +479,7 @@ func TestClient_BackupInvalidView(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(100, 1, 2, 3, SliceWidth-1)
s := test.NewServer()
defer s.Close()
@ -502,11 +502,11 @@ func TestClient_FragmentBlocks(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set two bits on blocks 0 & 3.
// Set two columns on blocks 0 & 3.
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(pilosa.HashBlockSize*3, 100)
// Set a bit on a different slice.
// Set a column on a different slice.
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
s := test.NewServer()

2
cluster.go
View file

@ -930,7 +930,7 @@ func (c *Cluster) Open() error {
// (and now in a state of STARTING) so that it can be put to the correct
// cluster state.
// TODO: Because the normal code path already sends a NodeJoin event (via
// memberlist), this it a bit redundant in most cases. Perhaps determine
// memberlist), this it a column redundant in most cases. Perhaps determine
// that the node has been restarted and don't do this step.
msg := &internal.NodeEventMessage{
Event: uint32(NodeJoin),

2
cluster_test.go
View file

@ -476,7 +476,7 @@ func TestCluster_ResizeStates(t *testing.T) {
t.Errorf("expected node1 topology: %v, but got: %v", expectedTop.NodeIDs, node1.Topology.NodeIDs)
}
// Bits
// Columns
// Verify that node-1 contains the fragment (i/f/standard/1) transferred from node-0.
node1Frame := node1.Holder.Frame("i", "f")
node1View := node1Frame.View("standard")

2
cmd/bench.go
View file

@ -45,7 +45,7 @@ Executes a benchmark for a given operation against the index.
flags.StringVarP(&Bencher.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Bencher.Index, "index", "i", "", "Pilosa index to benchmark.")
flags.StringVarP(&Bencher.Frame, "frame", "f", "", "Frame to benchmark.")
flags.StringVarP(&Bencher.Op, "operation", "o", "set-bit", "Operation to perform: choose from [set-bit]")
flags.StringVarP(&Bencher.Op, "operation", "o", "set-column", "Operation to perform: choose from [set-column]")
flags.IntVarP(&Bencher.N, "num", "n", 0, "Number of operations to perform.")
ctl.SetTLSConfig(flags, &Bencher.TLS.CertificatePath, &Bencher.TLS.CertificateKeyPath, &Bencher.TLS.SkipVerify)

4
cmd/bench_test.go
View file

@ -33,7 +33,7 @@ func TestBenchHelp(t *testing.T) {
func TestBenchConfig(t *testing.T) {
tests := []commandTest{
{
args: []string{"bench", "--operation", "set-bit"},
args: []string{"bench", "--operation", "set-column"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
@ -44,7 +44,7 @@ frame = "f1"
v.Check(cmd.Bencher.Host, "localhost:12345")
v.Check(cmd.Bencher.Index, "myindex")
v.Check(cmd.Bencher.Frame, "f1")
v.Check(cmd.Bencher.Op, "set-bit")
v.Check(cmd.Bencher.Op, "set-column")
v.Check(cmd.Bencher.N, 0)
return v.Error()
},

4
cmd/import.go
View file

@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
importCmd := &cobra.Command{
Use: "import",
Short: "Bulk load data into pilosa.",
Long: `Bulk imports one or more CSV files to a host's index and frame. The bits
Long: `Bulk imports one or more CSV files to a host's index and frame. The columns
of the CSV file are grouped by slice for the most efficient import.
The format of the CSV file is:
@ -57,7 +57,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.StringVarP(&Importer.Frame, "frame", "f", "", "Frame to import into.")
flags.StringVarP(&Importer.Field, "field", "", "", "Field to import into.")
flags.BoolVar(&Importer.StringKeys, "string-keys", false, "Treat payload as string keys.")
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of bits to buffer/sort before importing.")
flags.IntVarP(&Importer.BufferSize, "buffer-size", "s", 10000000, "Number of columns to buffer/sort before importing.")
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")

2
ctl/bench.go
View file

@ -62,7 +62,7 @@ func (cmd *BenchCommand) Run(ctx context.Context) error {
}
switch cmd.Op {
case "set-bit":
case "set-column":
return cmd.runSetBit(ctx, client)
case "":
return errors.New("op required")

2
ctl/bench_test.go
View file

@ -56,7 +56,7 @@ func TestBenchCommand_Run(t *testing.T) {
r, w, _ := os.Pipe()
cm := NewBenchCommand(stdin, w, w)
cm.Op = "set-bit"
cm.Op = "set-column"
cm.Host = "localhost:10101"
err := cm.Run(context.Background())

86
ctl/import.go
View file

@ -107,7 +107,7 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
// Import each path and import by slice.
for _, path := range cmd.Paths {
// Parse path into bits.
// Parse path into columns.
logger.Printf("parsing: %s", path)
if err := cmd.importPath(ctx, path); err != nil {
return err
@ -129,22 +129,22 @@ func (cmd *ImportCommand) ensureSchema(ctx context.Context) error {
return nil
}
// importPath parses a path into bits and imports it to the server.
// importPath parses a path into columns and imports it to the server.
func (cmd *ImportCommand) importPath(ctx context.Context, path string) error {
// If a field is provided, treat the import data as values to be range-encoded.
if cmd.Field != "" {
return cmd.bufferFieldValues(ctx, path)
} else {
if cmd.StringKeys {
return cmd.bufferBitsK(ctx, path)
return cmd.bufferColumnsK(ctx, path)
} else {
return cmd.bufferBits(ctx, path)
return cmd.bufferColumns(ctx, path)
}
}
}
// bufferBits buffers slices of bits to be imported as a batch.
func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
// bufferColumns buffers slices of columns to be imported as a batch.
func (cmd *ImportCommand) bufferColumns(ctx context.Context, path string) error {
a := make([]pilosa.Bit, 0, cmd.BufferSize)
var r *csv.Reader
@ -157,7 +157,7 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
}
defer f.Close()
// Read rows as bits.
// Read rows as columns.
r = csv.NewReader(f)
} else {
r = csv.NewReader(cmd.Stdin)
@ -183,21 +183,21 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
}
var bit pilosa.Bit
var column pilosa.Bit
// Parse row id.
rowID, err := strconv.ParseUint(record[0], 10, 64)
if err != nil {
return fmt.Errorf("invalid row id on row %d: %q", rnum, record[0])
}
bit.RowID = rowID
column.RowID = rowID
// Parse column id.
columnID, err := strconv.ParseUint(record[1], 10, 64)
if err != nil {
return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1])
}
bit.ColumnID = columnID
column.ColumnID = columnID
// Parse time, if exists.
if len(record) > 2 && record[2] != "" {
@ -205,44 +205,44 @@ func (cmd *ImportCommand) bufferBits(ctx context.Context, path string) error {
if err != nil {
return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2])
}
bit.Timestamp = t.UnixNano()
column.Timestamp = t.UnixNano()
}
a = append(a, bit)
a = append(a, column)
// If we've reached the buffer size then import bits.
// If we've reached the buffer size then import columns.
if len(a) == cmd.BufferSize {
if err := cmd.importBits(ctx, a); err != nil {
if err := cmd.importColumns(ctx, a); err != nil {
return err
}
a = a[:0]
}
}
// If there are still bits in the buffer then flush them.
if err := cmd.importBits(ctx, a); err != nil {
// If there are still columns in the buffer then flush them.
if err := cmd.importColumns(ctx, a); err != nil {
return err
}
return nil
}
// importBits sends batches of bits to the server.
func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) error {
// importColumns sends batches of columns to the server.
func (cmd *ImportCommand) importColumns(ctx context.Context, columns []pilosa.Bit) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// Group bits by slice.
logger.Printf("grouping %d bits", len(bits))
bitsBySlice := pilosa.Bits(bits).GroupBySlice()
// Group columns by slice.
logger.Printf("grouping %d columns", len(columns))
columnsBySlice := pilosa.Columns(columns).GroupBySlice()
// Parse path into bits.
for slice, bits := range bitsBySlice {
// Parse path into columns.
for slice, columns := range columnsBySlice {
if cmd.Sort {
sort.Sort(pilosa.BitsByPos(bits))
sort.Sort(pilosa.ColumnsByPos(columns))
}
logger.Printf("importing slice: %d, n=%d", slice, len(bits))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil {
logger.Printf("importing slice: %d, n=%d", slice, len(columns))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, columns); err != nil {
return errors.Wrap(err, "importing")
}
}
@ -250,8 +250,8 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
return nil
}
// bufferBitsK buffers slices of keys to be imported as a batch.
func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error {
// bufferColumnsK buffers slices of keys to be imported as a batch.
func (cmd *ImportCommand) bufferColumnsK(ctx context.Context, path string) error {
a := make([]pilosa.Bit, 0, cmd.BufferSize)
var r *csv.Reader
@ -264,7 +264,7 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error {
}
defer f.Close()
// Read rows as bits.
// Read rows as columns.
r = csv.NewReader(f)
} else {
r = csv.NewReader(cmd.Stdin)
@ -290,19 +290,19 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error {
return fmt.Errorf("bad column count on row %d: col=%d", rnum, len(record))
}
var bit pilosa.Bit
var column pilosa.Bit
// Parse row key.
if record[0] == "" {
return fmt.Errorf("invalid row key on row %d: %q", rnum, record[0])
}
bit.RowKey = record[0]
column.RowKey = record[0]
// Parse column key.
if record[1] == "" {
return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1])
}
bit.ColumnKey = record[1]
column.ColumnKey = record[1]
// Parse time, if exists.
if len(record) > 2 && record[2] != "" {
@ -310,36 +310,36 @@ func (cmd *ImportCommand) bufferBitsK(ctx context.Context, path string) error {
if err != nil {
return fmt.Errorf("invalid timestamp on row %d: %q", rnum, record[2])
}
bit.Timestamp = t.UnixNano()
column.Timestamp = t.UnixNano()
}
a = append(a, bit)
a = append(a, column)
// If we've reached the buffer size then import bits.
// If we've reached the buffer size then import columns.
if len(a) == cmd.BufferSize {
if err := cmd.importBitsK(ctx, a); err != nil {
if err := cmd.importColumnsK(ctx, a); err != nil {
return err
}
a = a[:0]
}
}
// If there are still bitKs in the buffer then flush them.
if err := cmd.importBitsK(ctx, a); err != nil {
// If there are still columnKs in the buffer then flush them.
if err := cmd.importColumnsK(ctx, a); err != nil {
return err
}
return nil
}
// importBitsK sends batches of bitKs to the server.
func (cmd *ImportCommand) importBitsK(ctx context.Context, bits []pilosa.Bit) error {
// importColumnsK sends batches of columnKs to the server.
func (cmd *ImportCommand) importColumnsK(ctx context.Context, columns []pilosa.Bit) error {
logger := log.New(cmd.Stderr, "", log.LstdFlags)
// TODO: does it help to sort the rowKeys?
logger.Printf("importing keys: n=%d", len(bits))
if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, bits); err != nil {
logger.Printf("importing keys: n=%d", len(columns))
if err := cmd.Client.ImportK(ctx, cmd.Index, cmd.Frame, columns); err != nil {
return errors.Wrap(err, "importing keys")
}
@ -360,7 +360,7 @@ func (cmd *ImportCommand) bufferFieldValues(ctx context.Context, path string) er
}
defer f.Close()
// Read rows as bits.
// Read rows as columns.
r = csv.NewReader(f)
} else {
r = csv.NewReader(cmd.Stdin)

12
executor.go
View file

@ -367,7 +367,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
}
}
if opt.ExcludeBits {
if opt.ExcludeColumns {
row.segments = []RowSegment{}
}
@ -1064,7 +1064,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel)
}
// Clear bits for each view.
// Clear columns for each view.
switch view {
case ViewStandard:
return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt)
@ -1164,7 +1164,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
timestamp = &t
}
// Set bits for each view.
// Set columns for each view.
switch view {
case ViewStandard:
return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt)
@ -1319,7 +1319,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
return err
}
frame.Stats.Count("SetBitmapAttrs", 1, 1.0)
frame.Stats.Count("SetColumnAttrs", 1, 1.0)
// Do not forward call if this is already being forwarded.
if opt.Remote {
@ -1404,7 +1404,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil {
return nil, err
}
frame.Stats.Count("SetBitmapAttrs", 1, 1.0)
frame.Stats.Count("SetColumnAttrs", 1, 1.0)
}
// Do not forward call if this is already being forwarded.
@ -1702,7 +1702,7 @@ type mapResponse struct {
type ExecOptions struct {
Remote bool
ExcludeAttrs bool
ExcludeBits bool
ExcludeColumns bool
}
// decodeError returns an error representation of s if s is non-blank.

176
executor_test.go
View file

@ -40,7 +40,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set bits.
// Set columns.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
@ -54,26 +54,26 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit bits.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil {
// Inhicolumn columns.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit attributes.
// Inhicolumn attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
@ -89,7 +89,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set bits.
// Set columns.
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, row=%d, col=%d)\n", 10, SliceWidth+1)+
@ -103,8 +103,8 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{10, 20}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
@ -115,17 +115,17 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
func TestExecutor_Execute_Difference(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 3)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 3)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 4)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -133,7 +133,7 @@ func TestExecutor_Execute_Difference(t *testing.T) {
func TestExecutor_Execute_Empty_Difference(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 1)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil {
@ -145,19 +145,19 @@ func TestExecutor_Execute_Empty_Difference(t *testing.T) {
func TestExecutor_Execute_Intersect(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -176,18 +176,18 @@ func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
func TestExecutor_Execute_Union(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -195,13 +195,13 @@ func TestExecutor_Execute_Union(t *testing.T) {
func TestExecutor_Execute_Empty_Union(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 0)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -209,18 +209,18 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
func TestExecutor_Execute_Xor(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetColumns(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetColumns(11, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -228,9 +228,9 @@ func TestExecutor_Execute_Xor(t *testing.T) {
func TestExecutor_Execute_Count(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetColumns(10, 3)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(10, SliceWidth+2)
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil {
@ -255,7 +255,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatal(err)
} else {
if !res[0].(bool) {
t.Fatalf("expected bit changed")
t.Fatalf("expected column changed")
}
}
@ -266,7 +266,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatal(err)
} else {
if res[0].(bool) {
t.Fatalf("expected bit unchanged")
t.Fatalf("expected column unchanged")
}
}
}
@ -409,7 +409,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set bits for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
@ -462,7 +462,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two slices.
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
@ -520,7 +520,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two slices.
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
@ -765,7 +765,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
t.Fatal(err)
}
// Set bits.
// Set columns.
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00")
SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00")
@ -784,8 +784,8 @@ func TestExecutor_Execute_Range(t *testing.T) {
t.Run("Standard", func(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
@ -793,8 +793,8 @@ func TestExecutor_Execute_Range(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 10}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
}
@ -854,7 +854,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("EQ", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -863,28 +863,28 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
// NEQ null
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ <int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ -<int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
//t.Fatalf("unexpected result: %s", spew.Sdump(result))
t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Bits())
t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Columns())
}
})
t.Run("LT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -892,7 +892,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("LTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -900,7 +900,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("GT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -908,7 +908,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("GTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -916,7 +916,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("BETWEEN", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -925,7 +925,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("FieldNotNull", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -933,7 +933,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("BelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -941,7 +941,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("AboveMax", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -949,16 +949,16 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("LTAboveMax", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo < 200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits()))
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns()))
}
})
t.Run("GTBelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo > -200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits()))
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns()))
}
})
@ -999,7 +999,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
t.Fatalf("unexpected slices: %+v", slices)
}
// Set bits in slice 0 & 2.
// Set columns in slice 0 & 2.
r := pilosa.NewRow(
(0*SliceWidth)+1,
(0*SliceWidth)+2,
@ -1013,13 +1013,13 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetColumns(10, (1*SliceWidth)+1)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -1047,8 +1047,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetColumns(10, (2*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetColumns(10, (2*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(row=10, frame=f))`), nil, nil); err != nil {
@ -1058,7 +1058,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
}
}
// Ensure a remote query can set bits on multiple nodes.
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
c := test.NewCluster(2)
c.ReplicaN = 2
@ -1101,7 +1101,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
t.Fatal(err)
}
// Verify that one bit is set on both node's holder.
// Verify that one column is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
@ -1110,7 +1110,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
}
// Ensure a remote query can set bits on multiple nodes.
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
c := test.NewCluster(2)
c.ReplicaN = 2
@ -1155,7 +1155,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
t.Fatal(err)
}
// Verify that one bit is set on both node's holder.
// Verify that one column is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
@ -1216,8 +1216,8 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.API.Holder = hldr.Holder
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetColumns(30, (2*SliceWidth)+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetColumns(30, (4*SliceWidth)+2)
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {

228
fragment.go
View file

@ -171,7 +171,7 @@ func (f *Fragment) Open() error {
// Clear checksums.
f.checksums = make(map[int][]byte)
// Read last bit to determine max row.
// Read last column to determine max row.
pos := f.storage.Max()
f.maxRowID = pos / SliceWidth
f.stats.Gauge("rows", float64(f.maxRowID), 1.0)
@ -380,7 +380,7 @@ func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *R
return row
}
// SetBit sets a bit for a given column & row within the fragment.
// SetBit sets a column for a given column & row within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
@ -390,10 +390,10 @@ func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) {
func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
changed = false
// Determine the position of the bit in the storage.
// Determine the position of the column in the storage.
pos, err := f.pos(rowID, columnID)
if err != nil {
return false, errors.Wrap(err, "getting bit ops")
return false, errors.Wrap(err, "getting column ops")
}
// Write to storage.
@ -432,7 +432,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
return changed, nil
}
// ClearBit clears a bit for a given column & row within the fragment.
// ClearBit clears a column for a given column & row within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) {
f.mu.Lock()
@ -442,10 +442,10 @@ func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) {
func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
changed = false
// Determine the position of the bit in the storage.
// Determine the position of the column in the storage.
pos, err := f.pos(rowID, columnID)
if err != nil {
return false, errors.Wrap(err, "getting bit pos")
return false, errors.Wrap(err, "getting column pos")
}
// Write to storage.
@ -478,7 +478,7 @@ func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
return changed, nil
}
func (f *Fragment) bit(rowID, columnID uint64) (bool, error) {
func (f *Fragment) column(rowID, columnID uint64) (bool, error) {
pos, err := f.pos(rowID, columnID)
if err != nil {
return false, err
@ -486,22 +486,22 @@ func (f *Fragment) bit(rowID, columnID uint64) (bool, error) {
return f.storage.Contains(pos), nil
}
// FieldValue uses a column of bits to read a multi-bit value.
func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
// FieldValue uses a column of columns to read a multi-column value.
func (f *Fragment) FieldValue(columnID uint64, columnDepth uint) (value uint64, exists bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
// If existence bit is unset then ignore remaining bits.
if v, err := f.bit(uint64(bitDepth), columnID); err != nil {
return 0, false, errors.Wrap(err, "getting existence bit")
// If existence column is unset then ignore remaining columns.
if v, err := f.column(uint64(columnDepth), columnID); err != nil {
return 0, false, errors.Wrap(err, "getting existence column")
} else if !v {
return 0, false, nil
}
// Compute other bits into a value.
for i := uint(0); i < bitDepth; i++ {
if v, err := f.bit(uint64(i), columnID); err != nil {
return 0, false, errors.Wrapf(err, "getting value bit %d", i)
// Compute other columns into a value.
for i := uint(0); i < columnDepth; i++ {
if v, err := f.column(uint64(i), columnID); err != nil {
return 0, false, errors.Wrapf(err, "getting value column %d", i)
} else if v {
value |= (1 << i)
}
@ -510,12 +510,12 @@ func (f *Fragment) FieldValue(columnID uint64, bitDepth uint) (value uint64, exi
return value, true, nil
}
// SetFieldValue uses a column of bits to set a multi-bit value.
func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
// SetFieldValue uses a column of columns to set a multi-column value.
func (f *Fragment) SetFieldValue(columnID uint64, columnDepth uint, value uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
for i := uint(0); i < bitDepth; i++ {
for i := uint(0); i < columnDepth; i++ {
if value&(1<<i) != 0 {
if c, err := f.setBit(uint64(i), columnID); err != nil {
return changed, err
@ -532,7 +532,7 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
}
// Mark value as set.
if c, err := f.setBit(uint64(bitDepth), columnID); err != nil {
if c, err := f.setBit(uint64(columnDepth), columnID); err != nil {
return changed, errors.Wrap(err, "marking not-null")
} else if c {
changed = true
@ -542,25 +542,25 @@ func (f *Fragment) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (
}
// importSetFieldValue is a more efficient SetFieldValue just for imports.
func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
func (f *Fragment) importSetFieldValue(columnID uint64, columnDepth uint, value uint64) (changed bool, err error) {
for i := uint(0); i < bitDepth; i++ {
for i := uint(0); i < columnDepth; i++ {
if value&(1<<i) != 0 {
bit, err := f.pos(uint64(i), columnID)
column, err := f.pos(uint64(i), columnID)
if err != nil {
return changed, errors.Wrap(err, "getting set pos")
}
if c, err := f.storage.Add(bit); err != nil {
if c, err := f.storage.Add(column); err != nil {
return changed, errors.Wrap(err, "adding")
} else if c {
changed = true
}
} else {
bit, err := f.pos(uint64(i), columnID)
column, err := f.pos(uint64(i), columnID)
if err != nil {
return changed, errors.Wrap(err, "getting clear pos")
}
if c, err := f.storage.Remove(bit); err != nil {
if c, err := f.storage.Remove(column); err != nil {
return changed, errors.Wrap(err, "removing")
} else if c {
changed = true
@ -569,7 +569,7 @@ func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uin
}
// Mark value as set.
p, err := f.pos(uint64(bitDepth), columnID)
p, err := f.pos(uint64(columnDepth), columnID)
if err != nil {
return changed, errors.Wrap(err, "marking not-null")
}
@ -584,23 +584,23 @@ func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uin
// FieldSum returns the sum of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
// Compute count based on the existence bit.
row := f.Row(uint64(bitDepth))
func (f *Fragment) FieldSum(filter *Row, columnDepth uint) (sum, count uint64, err error) {
// Compute count based on the existence column.
row := f.Row(uint64(columnDepth))
if filter != nil {
count = row.IntersectionCount(filter)
} else {
count = row.Count()
}
// Compute the sum based on the bit count of each row multiplied by the
// place value of each row. For example, 10 bits in the 1's place plus
// 4 bits in the 2's place plus 3 bits in the 4's place equals a total
// Compute the sum based on the column count of each row multiplied by the
// place value of each row. For example, 10 columns in the 1's place plus
// 4 columns in the 2's place plus 3 columns in the 4's place equals a total
// sum of 30:
//
// 10*(2^0) + 4*(2^1) + 3*(2^2) = 30
//
for i := uint(0); i < bitDepth; i++ {
for i := uint(0); i < columnDepth; i++ {
row := f.Row(uint64(i))
cnt := uint64(0)
if filter != nil {
@ -616,9 +616,9 @@ func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err
// FieldMin returns the min of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) {
func (f *Fragment) FieldMin(filter *Row, columnDepth uint) (min, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
consider := f.Row(uint64(columnDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
@ -628,8 +628,8 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err
return 0, 0, nil
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
for i := columnDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (columndepth-1) to 0
row := f.Row(uint64(ii))
x := consider.Difference(row)
@ -649,9 +649,9 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err
// FieldMax returns the max of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) {
func (f *Fragment) FieldMax(filter *Row, columnDepth uint) (max, count uint64, err error) {
consider := f.Row(uint64(bitDepth))
consider := f.Row(uint64(columnDepth))
if filter != nil {
consider = consider.Intersect(filter)
}
@ -661,8 +661,8 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err
return 0, 0, nil
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
for i := columnDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (columndepth-1) to 0
row := f.Row(uint64(ii))
x := row.Intersect(consider)
@ -679,31 +679,31 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err
}
// FieldRange returns bitmaps with a field value encoding matching the predicate.
func (f *Fragment) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
func (f *Fragment) FieldRange(op pql.Token, columnDepth uint, predicate uint64) (*Row, error) {
switch op {
case pql.EQ:
return f.fieldRangeEQ(bitDepth, predicate)
return f.fieldRangeEQ(columnDepth, predicate)
case pql.NEQ:
return f.fieldRangeNEQ(bitDepth, predicate)
return f.fieldRangeNEQ(columnDepth, predicate)
case pql.LT, pql.LTE:
return f.fieldRangeLT(bitDepth, predicate, op == pql.LTE)
return f.fieldRangeLT(columnDepth, predicate, op == pql.LTE)
case pql.GT, pql.GTE:
return f.fieldRangeGT(bitDepth, predicate, op == pql.GTE)
return f.fieldRangeGT(columnDepth, predicate, op == pql.GTE)
default:
return nil, ErrInvalidRangeOperation
}
}
func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
func (f *Fragment) fieldRangeEQ(columnDepth uint, predicate uint64) (*Row, error) {
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
b := f.Row(uint64(columnDepth))
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
// Filter any columns that don't match the current column value.
for i := int(columnDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
bit := (predicate >> uint(i)) & 1
column := (predicate >> uint(i)) & 1
if bit == 1 {
if column == 1 {
b = b.Intersect(row)
} else {
b = b.Difference(row)
@ -713,12 +713,12 @@ func (f *Fragment) fieldRangeEQ(bitDepth uint, predicate uint64) (*Row, error) {
return b, nil
}
func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error) {
func (f *Fragment) fieldRangeNEQ(columnDepth uint, predicate uint64) (*Row, error) {
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
b := f.Row(uint64(columnDepth))
// Get the equal bitmap.
eq, err := f.fieldRangeEQ(bitDepth, predicate)
eq, err := f.fieldRangeEQ(columnDepth, predicate)
if err != nil {
return nil, err
}
@ -729,21 +729,21 @@ func (f *Fragment) fieldRangeNEQ(bitDepth uint, predicate uint64) (*Row, error)
return b, nil
}
func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
func (f *Fragment) fieldRangeLT(columnDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
keep := NewRow()
// Start with set of columns with values set.
b := f.Row(uint64(bitDepth))
b := f.Row(uint64(columnDepth))
// Filter any bits that don't match the current bit value.
// Filter any columns that don't match the current column value.
leadingZeros := true
for i := int(bitDepth - 1); i >= 0; i-- {
for i := int(columnDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
bit := (predicate >> uint(i)) & 1
column := (predicate >> uint(i)) & 1
// Remove any columns with higher bits set.
// Remove any columns with higher columns set.
if leadingZeros {
if bit == 0 {
if column == 0 {
b = b.Difference(row)
continue
} else {
@ -751,23 +751,23 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b
}
}
// Handle last bit differently.
// If bit is zero then return only already kept columns.
// If bit is one then remove any one columns.
// Handle last column differently.
// If column is zero then return only already kept columns.
// If column is one then remove any one columns.
if i == 0 && !allowEquality {
if bit == 0 {
if column == 0 {
return keep, nil
}
return b.Difference(row.Difference(keep)), nil
}
// If bit is zero then remove all set columns not in excluded bitmap.
if bit == 0 {
// If column is zero then remove all set columns not in excluded bitmap.
if column == 0 {
b = b.Difference(row.Difference(keep))
continue
}
// If bit is set then add columns for set bits to exclude.
// If column is set then add columns for set columns to exclude.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep = keep.Union(b.Difference(row))
@ -777,32 +777,32 @@ func (f *Fragment) fieldRangeLT(bitDepth uint, predicate uint64, allowEquality b
return b, nil
}
func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
b := f.Row(uint64(bitDepth))
func (f *Fragment) fieldRangeGT(columnDepth uint, predicate uint64, allowEquality bool) (*Row, error) {
b := f.Row(uint64(columnDepth))
keep := NewRow()
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
// Filter any columns that don't match the current column value.
for i := int(columnDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
bit := (predicate >> uint(i)) & 1
column := (predicate >> uint(i)) & 1
// Handle last bit differently.
// If bit is one then return only already kept columns.
// If bit is zero then remove any unset columns.
// Handle last column differently.
// If column is one then return only already kept columns.
// If column is zero then remove any unset columns.
if i == 0 && !allowEquality {
if bit == 1 {
if column == 1 {
return keep, nil
}
return b.Difference(b.Difference(row).Difference(keep)), nil
}
// If bit is set then remove all unset columns not already kept.
if bit == 1 {
// If column is set then remove all unset columns not already kept.
if column == 1 {
b = b.Difference(b.Difference(row).Difference(keep))
continue
}
// If bit is unset then add columns with set bit to keep.
// If column is unset then add columns with set column to keep.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep = keep.Union(b.Intersect(row))
@ -812,29 +812,29 @@ func (f *Fragment) fieldRangeGT(bitDepth uint, predicate uint64, allowEquality b
return b, nil
}
// FieldNotNull returns the not-null row (stored at bitDepth).
func (f *Fragment) FieldNotNull(bitDepth uint) (*Row, error) {
return f.Row(uint64(bitDepth)), nil
// FieldNotNull returns the not-null row (stored at columnDepth).
func (f *Fragment) FieldNotNull(columnDepth uint) (*Row, error) {
return f.Row(uint64(columnDepth)), nil
}
// FieldRangeBetween returns bitmaps with a field value encoding matching any value between predicateMin and predicateMax.
func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
b := f.Row(uint64(bitDepth))
func (f *Fragment) FieldRangeBetween(columnDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
b := f.Row(uint64(columnDepth))
keep1 := NewRow() // GTE
keep2 := NewRow() // LTE
// Filter any bits that don't match the current bit value.
for i := int(bitDepth - 1); i >= 0; i-- {
// Filter any columns that don't match the current column value.
for i := int(columnDepth - 1); i >= 0; i-- {
row := f.Row(uint64(i))
bit1 := (predicateMin >> uint(i)) & 1
bit2 := (predicateMax >> uint(i)) & 1
column1 := (predicateMin >> uint(i)) & 1
column2 := (predicateMax >> uint(i)) & 1
// GTE predicateMin
// If bit is set then remove all unset columns not already kept.
if bit1 == 1 {
// If column is set then remove all unset columns not already kept.
if column1 == 1 {
b = b.Difference(b.Difference(row).Difference(keep1))
} else {
// If bit is unset then add columns with set bit to keep.
// If column is unset then add columns with set column to keep.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep1 = keep1.Union(b.Intersect(row))
@ -842,11 +842,11 @@ func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax u
}
// LTE predicateMin
// If bit is zero then remove all set columns not in excluded bitmap.
if bit2 == 0 {
// If column is zero then remove all set columns not in excluded bitmap.
if column2 == 0 {
b = b.Difference(row.Difference(keep2))
} else {
// If bit is set then add columns for set bits to exclude.
// If column is set then add columns for set columns to exclude.
// Don't bother to compute this on the final iteration.
if i > 0 {
keep2 = keep2.Union(b.Difference(row))
@ -867,7 +867,7 @@ func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
return Pos(rowID, columnID), nil
}
// ForEachBit executes fn for every bit set in the fragment.
// ForEachBit executes fn for every column set in the fragment.
// Errors returned from fn are passed through.
func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error {
f.mu.Lock()
@ -996,13 +996,13 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// If it's too low then don't try finding anymore pairs.
threshold := results.Pairs[0].Count
// If the row doesn't have enough bits set before the intersection
// If the row doesn't have enough columns set before the intersection
// then we can assume that any remaining rows also have a count too low.
if threshold < opt.MinThreshold || cnt < threshold {
break
}
// Calculate the intersecting bit count and skip if it's below our
// Calculate the intersecting column count and skip if it's below our
// last row in our current result set.
count := opt.Src.IntersectionCount(f.Row(rowID))
if count < threshold {
@ -1184,7 +1184,7 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i
}
}
// BlockData returns bits in a block as row & column ID pairs.
// BlockData returns columns in a block as row & column ID pairs.
func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) {
f.mu.Lock()
defer f.mu.Unlock()
@ -1196,11 +1196,11 @@ func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) {
return
}
// MergeBlock compares the block's bits and computes a diff with another set of block bits.
// The state of a bit is determined by consensus from all blocks being considered.
// MergeBlock compares the block's columns and computes a diff with another set of block columns.
// The state of a column is determined by consensus from all blocks being considered.
//
// For example, if 3 blocks are compared and two have a set bit and one has a
// cleared bit then the bit is considered cleared. The function returns the
// For example, if 3 blocks are compared and two have a set column and one has a
// cleared column then the column is considered cleared. The function returns the
// diff per incoming block so that all can be in sync.
func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) {
// Ensure that all pair sets are of equal length.
@ -1305,14 +1305,14 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
}
}
// Set local bits.
// Set local columns.
for i := range sets[0].ColumnIDs {
if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil {
return nil, nil, errors.Wrap(err, "setting")
}
}
// Clear local bits.
// Clear local columns.
for i := range clears[0].ColumnIDs {
if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil {
return nil, nil, errors.Wrap(err, "clearing")
@ -1322,7 +1322,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
return sets[1:], clears[1:], nil
}
// Import bulk imports a set of bits and then snapshots the storage.
// Import bulk imports a set of columns and then snapshots the storage.
// This does not affect the fragment's cache.
func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
f.mu.Lock()
@ -1335,7 +1335,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
// Disconnect op writer so we don't append updates.
f.storage.OpWriter = nil
// Process every bit.
// Process every column.
// If an error occurs then reopen the storage.
lastID := uint64(0)
if err := func() error {
@ -1343,10 +1343,10 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
for i := range rowIDs {
rowID, columnID := rowIDs[i], columnIDs[i]
// Determine the position of the bit in the storage.
// Determine the position of the column in the storage.
pos, err := f.pos(rowID, columnID)
if err != nil {
return errors.Wrap(err, "getting bit pos")
return errors.Wrap(err, "getting column pos")
}
// Write to storage.
@ -1393,7 +1393,7 @@ func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
}
// ImportValue bulk imports a set of range-encoded values.
func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error {
func (f *Fragment) ImportValue(columnIDs, values []uint64, columnDepth uint) error {
f.mu.Lock()
defer f.mu.Unlock()
// Verify that there are an equal number of column ids and values.
@ -1408,7 +1408,7 @@ func (f *Fragment) ImportValue(columnIDs, values []uint64, bitDepth uint) error
for i := range columnIDs {
columnID, value := columnIDs[i], values[i]
_, err := f.importSetFieldValue(columnID, bitDepth, value)
_, err := f.importSetFieldValue(columnID, columnDepth, value)
if err != nil {
return errors.Wrap(err, "setting")
}

334
fragment_test.go
View file

@ -38,12 +38,12 @@ var (
// SliceWidth is a helper reference to use when testing.
const SliceWidth = pilosa.SliceWidth
// Ensure a fragment can set a bit and retrieve it.
// Ensure a fragment can set a column and retrieve it.
func TestFragment_SetBit(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set bits on the fragment.
// Set columns on the fragment.
if _, err := f.SetBit(120, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(120, 6); err != nil {
@ -69,12 +69,12 @@ func TestFragment_SetBit(t *testing.T) {
}
}
// Ensure a fragment can clear a set bit.
// Ensure a fragment can clear a set column.
func TestFragment_ClearBit(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set and then clear bits on the fragment.
// Set and then clear columns on the fragment.
if _, err := f.SetBit(1000, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(1000, 2); err != nil {
@ -137,7 +137,7 @@ func TestFragment_SetFieldValue(t *testing.T) {
t.Fatal("expected change")
}
// Overwriting value should overwrite all bits.
// Overwriting value should overwrite all columns.
if changed, err := f.SetFieldValue(100, 16, 2028); err != nil {
t.Fatal(err)
} else if !changed {
@ -176,13 +176,13 @@ func TestFragment_SetFieldValue(t *testing.T) {
})
t.Run("QuickCheck", func(t *testing.T) {
if err := quick.Check(func(bitDepth uint, columnN uint64, values []uint64) bool {
// Limit bit depth & maximum values.
bitDepth = (bitDepth % 62) + 1
if err := quick.Check(func(columnDepth uint, columnN uint64, values []uint64) bool {
// Limit column depth & maximum values.
columnDepth = (columnDepth % 62) + 1
columnN = (columnN % 99) + 1
for i := range values {
values[i] = values[i] % (1 << bitDepth)
values[i] = values[i] % (1 << columnDepth)
}
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
@ -195,18 +195,18 @@ func TestFragment_SetFieldValue(t *testing.T) {
m[columnID] = int64(value)
if _, err := f.SetFieldValue(columnID, bitDepth, value); err != nil {
if _, err := f.SetFieldValue(columnID, columnDepth, value); err != nil {
t.Fatal(err)
}
}
// Ensure values are set.
for columnID, value := range m {
v, exists, err := f.FieldValue(columnID, bitDepth)
v, exists, err := f.FieldValue(columnID, columnDepth)
if err != nil {
t.Fatal(err)
} else if value != int64(v) {
t.Fatalf("value mismatch: column=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v)
t.Fatalf("value mismatch: column=%d, columndepth=%d, value: %d != %d", columnID, columnDepth, value, v)
} else if !exists {
t.Fatalf("value should exist: column=%d", columnID)
}
@ -221,24 +221,24 @@ func TestFragment_SetFieldValue(t *testing.T) {
// Ensure a fragment can sum field values.
func TestFragment_FieldSum(t *testing.T) {
const bitDepth = 16
const columnDepth = 16
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
} else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil {
t.Fatal(err)
}
t.Run("NoFilter", func(t *testing.T) {
if sum, n, err := f.FieldSum(nil, bitDepth); err != nil {
if sum, n, err := f.FieldSum(nil, columnDepth); err != nil {
t.Fatal(err)
} else if n != 4 {
t.Fatalf("unexpected count: %d", n)
@ -248,7 +248,7 @@ func TestFragment_FieldSum(t *testing.T) {
})
t.Run("WithFilter", func(t *testing.T) {
if sum, n, err := f.FieldSum(pilosa.NewRow(2000, 4000, 5000), bitDepth); err != nil {
if sum, n, err := f.FieldSum(pilosa.NewRow(2000, 4000, 5000), columnDepth); err != nil {
t.Fatal(err)
} else if n != 2 {
t.Fatalf("unexpected count: %d", n)
@ -260,25 +260,25 @@ func TestFragment_FieldSum(t *testing.T) {
// Ensure a fragment can find the min and max of field values.
func TestFragment_FieldMinMax(t *testing.T) {
const bitDepth = 16
const columnDepth = 16
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
} else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(5000, bitDepth, 2818); err != nil {
} else if _, err := f.SetFieldValue(5000, columnDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(6000, bitDepth, 2817); err != nil {
} else if _, err := f.SetFieldValue(6000, columnDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(7000, bitDepth, 0); err != nil {
} else if _, err := f.SetFieldValue(7000, columnDepth, 0); err != nil {
t.Fatal(err)
}
@ -296,7 +296,7 @@ func TestFragment_FieldMinMax(t *testing.T) {
{filter: pilosa.NewRow(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if min, cnt, err := f.FieldMin(test.filter, bitDepth); err != nil {
if min, cnt, err := f.FieldMin(test.filter, columnDepth); err != nil {
t.Fatal(err)
} else if min != test.exp {
t.Errorf("test %d expected min: %v, but got: %v", i, test.exp, min)
@ -320,7 +320,7 @@ func TestFragment_FieldMinMax(t *testing.T) {
{filter: pilosa.NewRow(7000), exp: 0, cnt: 1},
}
for i, test := range tests {
if max, cnt, err := f.FieldMax(test.filter, bitDepth); err != nil {
if max, cnt, err := f.FieldMax(test.filter, columnDepth); err != nil {
t.Fatal(err)
} else if max != test.exp {
t.Errorf("test %d expected max: %v, but got: %v", i, test.exp, max)
@ -333,28 +333,28 @@ func TestFragment_FieldMinMax(t *testing.T) {
// Ensure a fragment query for matching fields.
func TestFragment_FieldRange(t *testing.T) {
const bitDepth = 16
const columnDepth = 16
t.Run("EQ", func(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
} else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil {
t.Fatal(err)
}
// Query for equality.
if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil {
if b, err := f.FieldRange(pql.EQ, columnDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -363,21 +363,21 @@ func TestFragment_FieldRange(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2818); err != nil {
} else if _, err := f.SetFieldValue(3000, columnDepth, 2818); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(4000, columnDepth, 300); err != nil {
t.Fatal(err)
}
// Query for inequality.
if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil {
if b, err := f.FieldRange(pql.NEQ, columnDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -386,46 +386,46 @@ func TestFragment_FieldRange(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil {
} else if _, err := f.SetFieldValue(3000, columnDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil {
} else if _, err := f.SetFieldValue(4000, columnDepth, 301); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil {
} else if _, err := f.SetFieldValue(5000, columnDepth, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil {
} else if _, err := f.SetFieldValue(6000, columnDepth, 0); err != nil {
t.Fatal(err)
}
// Query for fields less than (ending with set bit).
if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil {
// Query for fields less than (ending with set column).
if b, err := f.FieldRange(pql.LT, columnDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields less than (ending with unset bit).
if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil {
// Query for fields less than (ending with unset column).
if b, err := f.FieldRange(pql.LT, columnDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields less than or equal to (ending with set bit).
if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil {
// Query for fields less than or equal to (ending with set column).
if b, err := f.FieldRange(pql.LTE, columnDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields less than or equal to (ending with unset bit).
if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil {
// Query for fields less than or equal to (ending with unset column).
if b, err := f.FieldRange(pql.LTE, columnDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -434,46 +434,46 @@ func TestFragment_FieldRange(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil {
} else if _, err := f.SetFieldValue(3000, columnDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil {
} else if _, err := f.SetFieldValue(4000, columnDepth, 301); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil {
} else if _, err := f.SetFieldValue(5000, columnDepth, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil {
} else if _, err := f.SetFieldValue(6000, columnDepth, 0); err != nil {
t.Fatal(err)
}
// Query for fields greater than (ending with unset bit).
if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil {
// Query for fields greater than (ending with unset column).
if b, err := f.FieldRange(pql.GT, columnDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than (ending with set bit).
if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil {
// Query for fields greater than (ending with set column).
if b, err := f.FieldRange(pql.GT, columnDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with unset bit).
if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil {
// Query for fields greater than or equal to (ending with unset column).
if b, err := f.FieldRange(pql.GTE, columnDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with set bit).
if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil {
// Query for fields greater than or equal to (ending with set column).
if b, err := f.FieldRange(pql.GTE, columnDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -482,46 +482,46 @@ func TestFragment_FieldRange(t *testing.T) {
defer f.Close()
// Set values.
if _, err := f.SetFieldValue(1000, bitDepth, 382); err != nil {
if _, err := f.SetFieldValue(1000, columnDepth, 382); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(2000, bitDepth, 300); err != nil {
} else if _, err := f.SetFieldValue(2000, columnDepth, 300); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(3000, bitDepth, 2817); err != nil {
} else if _, err := f.SetFieldValue(3000, columnDepth, 2817); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(4000, bitDepth, 301); err != nil {
} else if _, err := f.SetFieldValue(4000, columnDepth, 301); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(5000, bitDepth, 1); err != nil {
} else if _, err := f.SetFieldValue(5000, columnDepth, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetFieldValue(6000, bitDepth, 0); err != nil {
} else if _, err := f.SetFieldValue(6000, columnDepth, 0); err != nil {
t.Fatal(err)
}
// Query for fields greater than (ending with unset bit).
if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil {
// Query for fields greater than (ending with unset column).
if b, err := f.FieldRangeBetween(columnDepth, 300, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than (ending with set bit).
if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil {
// Query for fields greater than (ending with set column).
if b, err := f.FieldRangeBetween(columnDepth, 301, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with unset bit).
if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil {
// Query for fields greater than or equal to (ending with unset column).
if b, err := f.FieldRangeBetween(columnDepth, 301, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with set bit).
if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil {
// Query for fields greater than or equal to (ending with set column).
if b, err := f.FieldRangeBetween(columnDepth, 300, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
}
@ -531,7 +531,7 @@ func TestFragment_Snapshot(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set and then clear bits on the fragment.
// Set and then clear columns on the fragment.
if _, err := f.SetBit(1000, 1); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(1000, 2); err != nil {
@ -555,12 +555,12 @@ func TestFragment_Snapshot(t *testing.T) {
}
}
// Ensure a fragment can iterate over all bits in order.
// Ensure a fragment can iterate over all columns in order.
func TestFragment_ForEachBit(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set bits on the fragment.
// Set columns on the fragment.
if _, err := f.SetBit(100, 20); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(2, 38); err != nil {
@ -569,7 +569,7 @@ func TestFragment_ForEachBit(t *testing.T) {
t.Fatal(err)
}
// Iterate over bits.
// Iterate over columns.
var result [][2]uint64
if err := f.ForEachBit(func(rowID, columnID uint64) error {
result = append(result, [2]uint64{rowID, columnID})
@ -578,7 +578,7 @@ func TestFragment_ForEachBit(t *testing.T) {
t.Fatal(err)
}
// Verify bits are correct.
// Verify columns are correct.
if !reflect.DeepEqual(result, [][2]uint64{{2, 37}, {2, 38}, {100, 20}}) {
t.Fatalf("unexpected result: %#v", result)
}
@ -588,10 +588,10 @@ func TestFragment_ForEachBit(t *testing.T) {
func TestFragment_Top(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
// Set bits on the rows 100, 101, & 102.
f.MustSetBits(100, 1, 3, 200)
f.MustSetBits(101, 1)
f.MustSetBits(102, 1, 2)
// Set columns on the rows 100, 101, & 102.
f.MustSetColumns(100, 1, 3, 200)
f.MustSetColumns(101, 1)
f.MustSetColumns(102, 1, 2)
f.RecalculateCache()
// Retrieve top rows.
@ -611,10 +611,10 @@ func TestFragment_Top_Filter(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
// Set bits on the rows 100, 101, & 102.
f.MustSetBits(100, 1, 3, 200)
f.MustSetBits(101, 1)
f.MustSetBits(102, 1, 2)
// Set columns on the rows 100, 101, & 102.
f.MustSetColumns(100, 1, 3, 200)
f.MustSetColumns(101, 1)
f.MustSetColumns(102, 1, 2)
f.RecalculateCache()
// Assign attributes.
f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)})
@ -644,11 +644,11 @@ func TestFragment_TopN_Intersect(t *testing.T) {
// Create an intersecting input row.
src := pilosa.NewRow(1, 2, 3)
// Set bits on various rows.
f.MustSetBits(100, 1, 10, 11, 12) // one intersection
f.MustSetBits(101, 1, 2, 3, 4) // three intersections
f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections
f.MustSetBits(103, 1000, 1001, 1002) // no intersection
// Set columns on various rows.
f.MustSetColumns(100, 1, 10, 11, 12) // one intersection
f.MustSetColumns(101, 1, 2, 3, 4) // three intersections
f.MustSetColumns(102, 1, 2, 4, 5, 6) // two intersections
f.MustSetColumns(103, 1000, 1001, 1002) // no intersection
f.RecalculateCache()
// Retrieve top rows.
@ -663,7 +663,7 @@ func TestFragment_TopN_Intersect(t *testing.T) {
}
}
// Ensure a fragment can return top rows that have many bits set.
// Ensure a fragment can return top rows that have many columns set.
func TestFragment_TopN_Intersect_Large(t *testing.T) {
if testing.Short() {
t.Skip("short mode")
@ -678,10 +678,10 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
990, 991, 992, 993, 994, 995, 996, 997, 998, 999,
)
// Set bits on rows 0 - 999. Higher rows have higher bit counts.
// Set columns on rows 0 - 999. Higher rows have higher column counts.
for i := uint64(0); i < 1000; i++ {
for j := uint64(0); j < i; j++ {
f.MustSetBits(i, j)
f.MustSetColumns(i, j)
}
}
f.RecalculateCache()
@ -710,10 +710,10 @@ func TestFragment_TopN_IDs(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
// Set bits on various rows.
f.MustSetBits(100, 1, 2, 3)
f.MustSetBits(101, 4, 5, 6, 7)
f.MustSetBits(102, 8, 9, 10, 11, 12)
// Set columns on various rows.
f.MustSetColumns(100, 1, 2, 3)
f.MustSetColumns(101, 4, 5, 6, 7)
f.MustSetColumns(102, 8, 9, 10, 11, 12)
// Retrieve top rows.
if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil {
@ -731,10 +731,10 @@ func TestFragment_TopN_NopCache(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeNone)
defer f.Close()
// Set bits on various rows.
f.MustSetBits(100, 1, 2, 3)
f.MustSetBits(101, 4, 5, 6, 7)
f.MustSetBits(102, 8, 9, 10, 11, 12)
// Set columns on various rows.
f.MustSetColumns(100, 1, 2, 3)
f.MustSetColumns(101, 4, 5, 6, 7)
f.MustSetColumns(102, 8, 9, 10, 11, 12)
// Retrieve top rows.
if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil {
@ -783,13 +783,13 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
}
defer f.Close()
// Set bits on various rows.
f.MustSetBits(100, 1, 2, 3)
f.MustSetBits(101, 4, 5, 6, 7)
f.MustSetBits(102, 8, 9, 10, 11, 12)
f.MustSetBits(103, 8, 9, 10, 11, 12, 13)
f.MustSetBits(104, 8, 9, 10, 11, 12, 13, 14)
f.MustSetBits(105, 10, 11)
// Set columns on various rows.
f.MustSetColumns(100, 1, 2, 3)
f.MustSetColumns(101, 4, 5, 6, 7)
f.MustSetColumns(102, 8, 9, 10, 11, 12)
f.MustSetColumns(103, 8, 9, 10, 11, 12, 13)
f.MustSetColumns(104, 8, 9, 10, 11, 12, 13, 14)
f.MustSetColumns(105, 10, 11)
f.RecalculateCache()
@ -816,7 +816,7 @@ func TestFragment_Checksum(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Retrieve checksum and set bits.
// Retrieve checksum and set columns.
orig := f.Checksum()
if _, err := f.SetBit(1, 200); err != nil {
t.Fatal(err)
@ -838,7 +838,7 @@ func TestFragment_Blocks(t *testing.T) {
// Retrieve initial checksum.
var prev []pilosa.FragmentBlock
// Set first bit.
// Set first column.
if _, err := f.SetBit(0, 0); err != nil {
t.Fatal(err)
}
@ -848,7 +848,7 @@ func TestFragment_Blocks(t *testing.T) {
}
prev = blocks
// Set bit on different row.
// Set column on different row.
if _, err := f.SetBit(20, 0); err != nil {
t.Fatal(err)
}
@ -858,7 +858,7 @@ func TestFragment_Blocks(t *testing.T) {
}
prev = blocks
// Set bit on different column.
// Set column on different column.
if _, err := f.SetBit(20, 100); err != nil {
t.Fatal(err)
}
@ -873,7 +873,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set bits on a different block.
// Set columns on a different block.
if _, err := f.SetBit(100, 1); err != nil {
t.Fatal(err)
}
@ -891,7 +891,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU)
defer f.Close()
// Set bits on the fragment.
// Set columns on the fragment.
for i := uint64(0); i < 1000; i++ {
if _, err := f.SetBit(i, 0); err != nil {
t.Fatal(err)
@ -941,7 +941,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
t.Fatal(err)
}
// Set bits on the fragment.
// Set columns on the fragment.
for i := uint64(0); i < 1000; i++ {
if _, err := f.SetBit(i, 0); err != nil {
t.Fatal(err)
@ -976,7 +976,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
f0 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f0.Close()
// Set and then clear bits on the fragment.
// Set and then clear columns on the fragment.
if _, err := f0.SetBit(1000, 1); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit(1000, 2); err != nil {
@ -1011,8 +1011,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
}
// Verify data in other fragment.
if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected columns: %+v", a)
}
// Close and reopen the fragment & verify the data.
@ -1020,8 +1020,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
t.Fatal(err)
} else if n := f1.Cache().Len(); n != 1 {
t.Fatalf("unexpected cache size (reopen): %d", n)
} else if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected bits (reopen): %+v", a)
} else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected columns (reopen): %+v", a)
}
}
@ -1083,10 +1083,10 @@ func TestFragment_Tanimoto(t *testing.T) {
src := pilosa.NewRow(1, 2, 3)
// Set bits on the rows 100, 101, & 102.
f.MustSetBits(100, 1, 3, 2, 200)
f.MustSetBits(101, 1, 3)
f.MustSetBits(102, 1, 2, 10, 12)
// Set columns on the rows 100, 101, & 102.
f.MustSetColumns(100, 1, 3, 2, 200)
f.MustSetColumns(101, 1, 3)
f.MustSetColumns(102, 1, 2, 10, 12)
f.RecalculateCache()
if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 50, Src: src}); err != nil {
@ -1106,10 +1106,10 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
src := pilosa.NewRow(1, 2, 3)
// Set bits on the rows 100, 101, & 102.
f.MustSetBits(100, 1, 3, 2, 200)
f.MustSetBits(101, 1, 3)
f.MustSetBits(102, 1, 2, 10, 12)
// Set columns on the rows 100, 101, & 102.
f.MustSetColumns(100, 1, 3, 2, 200)
f.MustSetColumns(101, 1, 3)
f.MustSetColumns(102, 1, 2, 10, 12)
f.RecalculateCache()
if pairs, err := f.Top(pilosa.TopOptions{TanimotoThreshold: 0, Src: src}); err != nil {
@ -1129,7 +1129,7 @@ func TestFragment_Snapshot_Run(t *testing.T) {
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set bits on the fragment.
// Set columns on the fragment.
for i := uint64(1); i < 3; i++ {
if _, err := f.SetBit(1000, i); err != nil {
t.Fatal(err)

24
frame.go
View file

@ -587,7 +587,7 @@ func (f *Frame) DeleteView(name string) error {
return nil
}
// SetBit sets a bit on a view within the frame.
// SetBit sets a column on a view within the frame.
func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Validate view name.
if !IsValidView(name) {
@ -600,7 +600,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed
return changed, errors.Wrap(err, "creating view")
}
// Set non-time bit.
// Set non-time column.
if v, err := view.SetBit(rowID, colID); err != nil {
return changed, errors.Wrap(err, "setting on view")
} else if v {
@ -612,7 +612,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed
return changed, nil
}
// If a timestamp is specified then set bits across all views for the quantum.
// If a timestamp is specified then set columns across all views for the quantum.
for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
if err != nil {
@ -629,7 +629,7 @@ func (f *Frame) SetBit(name string, rowID, colID uint64, t *time.Time) (changed
return changed, nil
}
// ClearBit clears a bit within the frame.
// ClearBit clears a column within the frame.
func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (changed bool, err error) {
// Validate view name.
if !IsValidView(name) {
@ -642,7 +642,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
return changed, errors.Wrap(err, "creating view")
}
// Clear non-time bit.
// Clear non-time column.
if v, err := view.ClearBit(rowID, colID); err != nil {
return changed, errors.Wrap(err, "setting on view")
} else if v {
@ -654,7 +654,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
return changed, nil
}
// If a timestamp is specified then clear bits across all views for the quantum.
// If a timestamp is specified then clear columns across all views for the quantum.
for _, subname := range ViewsByTime(name, *t, f.TimeQuantum()) {
view, err := f.CreateViewIfNotExists(subname)
if err != nil {
@ -846,13 +846,13 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
inverse = []string{ViewInverse}
} else {
standard = ViewsByTime(ViewStandard, *timestamp, q)
// In order to match the logic of `SetBit()`, we want bits
// In order to match the logic of `SetBit()`, we want columns
// with timestamps to write to both time and standard views.
standard = append(standard, ViewStandard)
inverse = ViewsByTime(ViewInverse, *timestamp, q)
}
// Attach bit to each standard view.
// Attach column to each standard view.
for _, name := range standard {
key := importKey{View: name, Slice: columnID / SliceWidth}
data := dataByFragment[key]
@ -862,7 +862,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
}
if f.inverseEnabled {
// Attach reversed bits to each inverse view.
// Attach reversed columns to each inverse view.
for _, name := range inverse {
key := importKey{View: name, Slice: rowID / SliceWidth}
data := dataByFragment[key]
@ -909,7 +909,7 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
// ImportValue bulk imports range-encoded value data.
func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64) error {
viewName := ViewFieldPrefix + fieldName
// Get the field so we know bitDepth.
// Get the field so we know columnDepth.
field := f.Field(fieldName)
if field == nil {
return fmt.Errorf("Field does not exist: %s", fieldName)
@ -939,7 +939,7 @@ func (f *Frame) ImportValue(fieldName string, columnIDs []uint64, values []int64
for key, data := range dataByFragment {
// The view must already exist (i.e. we can't create it)
// because we need to know bitDepth (based on min/max value).
// because we need to know columnDepth (based on min/max value).
view, err := f.CreateViewIfNotExists(key.View)
if err != nil {
return errors.Wrap(err, "creating view")
@ -1064,7 +1064,7 @@ type Field struct {
Max int64 `json:"max,omitempty"`
}
// BitDepth returns the number of bits required to store a value between min & max.
// BitDepth returns the number of columns required to store a value between min & max.
func (f *Field) BitDepth() uint {
for i := uint(0); i < 63; i++ {
if f.Max-f.Min < (1 << i) {

10
handler.go
View file

@ -82,7 +82,7 @@ func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")
h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse")
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits")
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeColumns")
h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice")
@ -825,7 +825,7 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
Slices: slices,
ColumnAttrs: q.Get("columnAttrs") == "true",
ExcludeAttrs: q.Get("excludeAttrs") == "true",
ExcludeBits: q.Get("excludeBits") == "true",
ExcludeColumns: q.Get("excludeColumns") == "true",
}, nil
}
@ -1183,8 +1183,8 @@ type QueryRequest struct {
// Do not return row attributes, if true.
ExcludeAttrs bool
// Do not return bits, if true.
ExcludeBits bool
// Do not return columns, if true.
ExcludeColumns bool
// If true, indicates that query is part of a larger distributed query.
// If false, this request is on the originating node.
@ -1198,7 +1198,7 @@ func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest {
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeAttrs: pb.ExcludeAttrs,
ExcludeBits: pb.ExcludeBits,
ExcludeColumns: pb.ExcludeColumns,
}
return req

32
handler_test.go
View file

@ -194,13 +194,13 @@ func TestHandler_MaxSlices(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetBits(30, (3*SliceWidth)+4)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetColumns(30, (1*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetColumns(30, (1*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 3).MustSetColumns(30, (3*SliceWidth)+4)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetColumns(40, (0*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetColumns(40, (0*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetColumns(40, (0*SliceWidth)+8)
h := test.NewHandler()
h.API.Holder = hldr.Holder
@ -419,7 +419,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -452,7 +452,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -484,8 +484,8 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
@ -541,8 +541,8 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
@ -1110,9 +1110,9 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set bits in the index.
// Set columns in the index.
f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
f0.MustSetBits(100, 1, 2, 3)
f0.MustSetColumns(100, 1, 2, 3)
// Begin backing up from slice i/f/0.
resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0")
@ -1145,8 +1145,8 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0)
if f1 == nil {
t.Fatal("fragment x/y/standard/0 not created")
} else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) {
t.Fatalf("unexpected restored bits: %+v", bits)
} else if columns := f1.Row(100).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 3}) {
t.Fatalf("unexpected restored columns: %+v", columns)
}
}

2
holder.go
View file

@ -466,7 +466,7 @@ func (h *Holder) flushCaches() {
// RecalculateCaches recalculates caches on every index in the holder. This is
// probably not practical to call in real-world workloads, but makes writing
// integration tests much eaiser, since one doesn't have to wait 10 seconds
// after setting bits to get expected response.
// after setting columns to get expected response.
func (h *Holder) RecalculateCaches() {
for _, index := range h.Indexes() {
index.RecalculateCaches()

90
holder_test.go
View file

@ -330,7 +330,7 @@ func TestHolder_DeleteIndex(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Write bits to separate indexes.
// Write columns to separate indexes.
f0 := hldr.MustCreateFragmentIfNotExists("i0", "f", pilosa.ViewStandard, 0)
if _, err := f0.SetBit(100, 200); err != nil {
t.Fatal(err)
@ -456,29 +456,29 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
}
@ -554,29 +554,29 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
@ -597,16 +597,16 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
@ -615,8 +615,8 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
}

42
internal/public.pb.go
View file

@ -43,7 +43,7 @@ var _ = math.Inf
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type Row struct {
Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"`
Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"`
Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
}
@ -53,9 +53,9 @@ func (m *Row) String() string { return proto.CompactTextString(m) }
func (*Row) ProtoMessage() {}
func (*Row) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{0} }
func (m *Row) GetBits() []uint64 {
func (m *Row) GetColumns() []uint64 {
if m != nil {
return m.Bits
return m.Columns
}
return nil
}
@ -272,7 +272,7 @@ type QueryRequest struct {
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
ExcludeAttrs bool `protobuf:"varint,6,opt,name=ExcludeAttrs,proto3" json:"ExcludeAttrs,omitempty"`
ExcludeBits bool `protobuf:"varint,7,opt,name=ExcludeBits,proto3" json:"ExcludeBits,omitempty"`
ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"`
}
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
@ -315,9 +315,9 @@ func (m *QueryRequest) GetExcludeAttrs() bool {
return false
}
func (m *QueryRequest) GetExcludeBits() bool {
func (m *QueryRequest) GetExcludeColumns() bool {
if m != nil {
return m.ExcludeBits
return m.ExcludeColumns
}
return false
}
@ -575,10 +575,10 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.Bits) > 0 {
dAtA2 := make([]byte, len(m.Bits)*10)
if len(m.Columns) > 0 {
dAtA2 := make([]byte, len(m.Columns)*10)
var j1 int
for _, num := range m.Bits {
for _, num := range m.Columns {
for num >= 1<<7 {
dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -808,7 +808,7 @@ func (m *Attr) MarshalTo(dAtA []byte) (int, error) {
if m.FloatValue != 0 {
dAtA[i] = 0x31
i++
binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64bits(float64(m.FloatValue))))
binary.LittleEndian.PutUint64(dAtA[i:], uint64(math.Float64columns(float64(m.FloatValue))))
i += 8
}
return i, nil
@ -912,10 +912,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.ExcludeBits {
if m.ExcludeColumns {
dAtA[i] = 0x38
i++
if m.ExcludeBits {
if m.ExcludeColumns {
dAtA[i] = 1
} else {
dAtA[i] = 0
@ -1263,9 +1263,9 @@ func encodeVarintPublic(dAtA []byte, offset int, v uint64) int {
func (m *Row) Size() (n int) {
var l int
_ = l
if len(m.Bits) > 0 {
if len(m.Columns) > 0 {
l = 0
for _, e := range m.Bits {
for _, e := range m.Columns {
l += sovPublic(uint64(e))
}
n += 1 + sovPublic(uint64(l)) + l
@ -1408,7 +1408,7 @@ func (m *QueryRequest) Size() (n int) {
if m.ExcludeAttrs {
n += 2
}
if m.ExcludeBits {
if m.ExcludeColumns {
n += 2
}
return n
@ -1615,7 +1615,7 @@ func (m *Row) Unmarshal(dAtA []byte) error {
break
}
}
m.Bits = append(m.Bits, v)
m.Columns = append(m.Columns, v)
} else if wireType == 2 {
var packedLen int
for shift := uint(0); ; shift += 7 {
@ -1655,10 +1655,10 @@ func (m *Row) Unmarshal(dAtA []byte) error {
break
}
}
m.Bits = append(m.Bits, v)
m.Columns = append(m.Columns, v)
}
} else {
return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Columns", wireType)
}
case 2:
if wireType != 2 {
@ -2337,7 +2337,7 @@ func (m *Attr) Unmarshal(dAtA []byte) error {
}
v = uint64(binary.LittleEndian.Uint64(dAtA[iNdEx:]))
iNdEx += 8
m.FloatValue = float64(math.Float64frombits(v))
m.FloatValue = float64(math.Float64fromcolumns(v))
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -2622,7 +2622,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
m.ExcludeAttrs = bool(v != 0)
case 7:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeBits", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeColumns", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
@ -2639,7 +2639,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.ExcludeBits = bool(v != 0)
m.ExcludeColumns = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])

40
row.go
View file

@ -22,7 +22,7 @@ import (
"github.com/pilosa/pilosa/roaring"
)
// Row represents a set of bits.
// Row represents a set of columns.
type Row struct {
segments []RowSegment
@ -31,9 +31,9 @@ type Row struct {
}
// NewRow returns a new instance of Row.
func NewRow(bits ...uint64) *Row {
func NewRow(columns ...uint64) *Row {
r := &Row{}
for _, i := range bits {
for _, i := range columns {
r.SetBit(i)
}
return r
@ -115,7 +115,7 @@ func (r *Row) Xor(other *Row) *Row {
return &Row{segments: segments}
}
// Union returns the bitwise union of r and other.
// Union returns the columnwise union of r and other.
func (r *Row) Union(other *Row) *Row {
var segments []RowSegment
itr := newMergeSegmentIterator(r.segments, other.segments)
@ -151,12 +151,12 @@ func (r *Row) Difference(other *Row) *Row {
return &Row{segments: segments}
}
// SetBit sets the i-th bit of the row.
// SetBit sets the i-th column of the row.
func (r *Row) SetBit(i uint64) (changed bool) {
return r.createSegmentIfNotExists(i / SliceWidth).SetBit(i)
}
// ClearBit clears the i-th bit of the row.
// ClearBit clears the i-th column of the row.
func (r *Row) ClearBit(i uint64) (changed bool) {
s := r.segment(i / SliceWidth)
if s == nil {
@ -226,7 +226,7 @@ func (r *Row) DecrementCount(i uint64) {
}
}
// Count returns the number of set bits in the row.
// Count returns the number of set columns in the row.
func (r *Row) Count() uint64 {
var n uint64
for i := range r.segments {
@ -239,9 +239,9 @@ func (r *Row) Count() uint64 {
func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
Attrs map[string]interface{} `json:"attrs"`
Bits []uint64 `json:"bits"`
Columns []uint64 `json:"columns"`
}
o.Bits = r.Bits()
o.Columns = r.Columns()
o.Attrs = r.Attrs
if o.Attrs == nil {
@ -251,11 +251,11 @@ func (r *Row) MarshalJSON() ([]byte, error) {
return json.Marshal(&o)
}
// Bits returns the bits in r as a slice of ints.
func (r *Row) Bits() []uint64 {
// Columns returns the columns in r as a slice of ints.
func (r *Row) Columns() []uint64 {
a := make([]uint64, 0, r.Count())
for i := range r.segments {
a = append(a, r.segments[i].Bits()...)
a = append(a, r.segments[i].Columns()...)
}
return a
}
@ -267,7 +267,7 @@ func encodeRow(r *Row) *internal.Row {
}
return &internal.Row{
Bits: r.Bits(),
Columns: r.Columns(),
Attrs: encodeAttrs(r.Attrs),
}
}
@ -280,7 +280,7 @@ func decodeRow(pr *internal.Row) *Row {
r := NewRow()
r.Attrs = decodeAttrs(pr.Attrs)
for _, v := range pr.Bits {
for _, v := range pr.Columns {
r.SetBit(v)
}
return r
@ -339,7 +339,7 @@ func (s *RowSegment) Intersect(other *RowSegment) *RowSegment {
}
}
// Union returns the bitwise union of s and other.
// Union returns the columnwise union of s and other.
func (s *RowSegment) Union(other *RowSegment) *RowSegment {
data := s.data.Union(&other.data)
@ -372,7 +372,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment {
}
}
// SetBit sets the i-th bit of the row.
// SetBit sets the i-th column of the row.
func (s *RowSegment) SetBit(i uint64) (changed bool) {
s.ensureWritable()
changed, _ = s.data.Add(i)
@ -382,7 +382,7 @@ func (s *RowSegment) SetBit(i uint64) (changed bool) {
return changed
}
// ClearBit clears the i-th bit of the row.
// ClearBit clears the i-th column of the row.
func (s *RowSegment) ClearBit(i uint64) (changed bool) {
s.ensureWritable()
@ -398,8 +398,8 @@ func (s *RowSegment) InvalidateCount() {
s.n = s.data.Count()
}
// Bits returns a list of all bits set in the segment.
func (s *RowSegment) Bits() []uint64 {
// Columns returns a list of all columns set in the segment.
func (s *RowSegment) Columns() []uint64 {
a := make([]uint64, 0, s.Count())
itr := s.data.Iterator()
for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
@ -408,7 +408,7 @@ func (s *RowSegment) Bits() []uint64 {
return a
}
// Count returns the number of set bits in the row.
// Count returns the number of set columns in the row.
func (s *RowSegment) Count() uint64 { return s.n }
// ensureWritable clones the segment if it is pointing to non-writable data.

22
row_test.go
View file

@ -47,7 +47,7 @@ func TestRow_Merge(t *testing.T) {
if cnt := test.r1.Count(); cnt != test.exp {
t.Fatalf("merged count %d is not %d", cnt, test.exp)
}
if length := len(test.r1.Bits()); uint64(length) != test.exp {
if length := len(test.r1.Columns()); uint64(length) != test.exp {
t.Fatalf("merged length %d is not %d", length, test.exp)
}
})
@ -65,15 +65,15 @@ func TestRow_Xor(t *testing.T) {
t.Fatalf("Test 1 Count after xor %d != 3\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Results %v != expected %v\n", res.Columns(), exp)
}
res = r2.Xor(r1)
if res.Count() != 3 {
t.Fatalf("Test 3 Count after xor %d != 3\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 4 Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 4 Results %v != expected %v\n", res.Columns(), exp)
}
}
@ -86,15 +86,15 @@ func TestRow_Union_Segment(t *testing.T) {
if res.Count() != 4 {
t.Fatalf("Test 1 Count after Union %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Columns(), exp)
}
res = r2.Union(r1)
if res.Count() != 4 {
t.Fatalf("Test 3 Count after xor %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Columns(), exp)
}
}
@ -107,7 +107,7 @@ func TestRow_Difference_Segment(t *testing.T) {
if res.Count() != 2 {
t.Fatalf("Test 1 Count after Difference %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Columns(), exp)
}
}

6
server/cluster_test.go
View file

@ -464,12 +464,12 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// This is an attempt to ensure there is data on both nodes, but is not guaranteed.
// TODO: Deterministic node IDs would ensure consistent results
setBits := ""
setColumns := ""
for i := 0; i < 20; i++ {
setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth)
setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setBits); err != nil {
if _, err := m0.Query("i", "", setColumns); err != nil {
t.Fatal(err)
}

40
server/server_test.go
View file

@ -69,7 +69,7 @@ func TestMain_Set_Quick(t *testing.T) {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"bits": columnIDs,
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
@ -92,7 +92,7 @@ func TestMain_Set_Quick(t *testing.T) {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"bits": columnIDs,
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
@ -132,7 +132,7 @@ func TestMain_SetRowAttrs(t *testing.T) {
t.Fatal(err)
}
// Set bits on different rows in different frames.
// Set columns on different rows in different frames.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil {
@ -157,14 +157,14 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Query row x/1.
if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -175,19 +175,19 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Query rows after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
@ -205,7 +205,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
t.Fatal(err)
}
// Set bits on row.
// Set columns on row.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil {
@ -220,7 +220,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
// Query row.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -231,7 +231,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
// Query row after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}
@ -266,7 +266,7 @@ func TestMain_InverseSlices(t *testing.T) {
SetBit(col=1, frame="f", row=2000)
SetBit(col=1, frame="f", row=%d)
`, 1*pilosa.SliceWidth)); err != nil {
t.Fatal("setting bits:", err)
t.Fatal("setting columns:", err)
}
time.Sleep(1 * time.Second)
@ -274,12 +274,12 @@ func TestMain_InverseSlices(t *testing.T) {
// Query the cluster.
if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != fmt.Sprintf(`{"results":[{"attrs":{},"bits":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" {
} else if res != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
// Ensure program can set bits on one cluster and then restore to a second cluster.
// Ensure program can set columns on one cluster and then restore to a second cluster.
func TestMain_FrameRestore(t *testing.T) {
mains1 := test.MustRunMainWithCluster(t, 2)
m10 := mains1[0]
@ -304,13 +304,13 @@ func TestMain_FrameRestore(t *testing.T) {
SetBit(row=1, frame="f", col=600000)
SetBit(row=1, frame="f", col=800000)
`); err != nil {
t.Fatal("setting bits:", err)
t.Fatal("setting columns:", err)
}
// Query row on first cluster.
if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -348,7 +348,7 @@ func TestMain_FrameRestore(t *testing.T) {
// Query row on second cluster.
if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("2unexpected result: %s", res)
}
}
@ -408,7 +408,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
t.Fatal("create frame:", err)
}
// Set some bits
// Set some columns
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
@ -416,7 +416,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
}
}
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {
t.Fatal("setting bits:", err)
t.Fatal("setting columns:", err)
}
// Calculate caches on the first node
@ -440,7 +440,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
}
}
// SetCommand represents a command to set a bit.
// SetCommand represents a command to set a column.
type SetCommand struct {
ID uint64
Frame string

8
stats_test.go
View file

@ -78,7 +78,7 @@ func TestMultiStatClient_Expvar(t *testing.T) {
t.Fatalf("unexpected expvar : %s", pilosa.Expvar.String())
}
// Expvar should ignore earlier set tags from setbit
// Expvar should ignore earlier set tags from setcolumn
if hldr.Stats.Tags() != nil {
t.Fatalf("unexpected tag")
}
@ -146,7 +146,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
}
}
func TestStatsCount_SetBitmapAttrs(t *testing.T) {
func TestStatsCount_SetColumnAttrs(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -162,8 +162,8 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) {
frame.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "SetBitmapAttrs" {
t.Errorf("Expected SetBitmapAttrs, Results %s", name)
if name != "SetColumnAttrs" {
t.Errorf("Expected SetColumnAttrs, Results %s", name)
}
called = true
},

10
test/fragment.go
View file

@ -85,9 +85,9 @@ func (f *Fragment) Reopen() error {
return nil
}
// MustSetBits sets bits on a row. Panic on error.
// MustSetColumns sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
func (f *Fragment) MustSetColumns(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.SetBit(rowID, columnID); err != nil {
panic(err)
@ -95,8 +95,8 @@ func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
}
}
// MustClearBits clears bits on a row. Panic on error.
func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) {
// MustClearColumns clears columns on a row. Panic on error.
func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.ClearBit(rowID, columnID); err != nil {
panic(err)
@ -126,7 +126,7 @@ func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) {
s.attrs[id] = m
}
// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk.
// GenerateImportFill generates a set of columns pairs that evenly fill a fragment chunk.
func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
ipct := int(pct * 100)
for i := 0; i < SliceWidth*rowN; i++ {

2
test/frame.go
View file

@ -75,7 +75,7 @@ func (f *Frame) Reopen() error {
return nil
}
// MustSetBit sets a bit on the frame. Panic on error.
// MustSetBit sets a column on the frame. Panic on error.
func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) {
changed, err := f.SetBit(view, rowID, columnID, t)
if err != nil {

2
time_test.go
View file

@ -65,7 +65,7 @@ func TestViewByTimeUnit(t *testing.T) {
})
}
// Ensure all applicable frame names can be generated when mutating a time bit.
// Ensure all applicable frame names can be generated when mutating a time column.
func TestViewsByTime(t *testing.T) {
ts := time.Date(2000, time.January, 2, 3, 4, 5, 6, time.UTC)

36
view.go
View file

@ -305,7 +305,7 @@ func (v *View) DeleteFragment(slice uint64) error {
return nil
}
// SetBit sets a bit within the view.
// SetBit sets a column within the view.
func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
@ -315,7 +315,7 @@ func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) {
return frag.SetBit(rowID, columnID)
}
// ClearBit clears a bit within the view.
// ClearBit clears a column within the view.
func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
@ -325,30 +325,30 @@ func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
return frag.ClearBit(rowID, columnID)
}
// FieldValue uses a column of bits to read a multi-bit value.
func (v *View) FieldValue(columnID uint64, bitDepth uint) (value uint64, exists bool, err error) {
// FieldValue uses a column of columns to read a multi-column value.
func (v *View) FieldValue(columnID uint64, columnDepth uint) (value uint64, exists bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return value, exists, err
}
return frag.FieldValue(columnID, bitDepth)
return frag.FieldValue(columnID, columnDepth)
}
// SetFieldValue uses a column of bits to set a multi-bit value.
func (v *View) SetFieldValue(columnID uint64, bitDepth uint, value uint64) (changed bool, err error) {
// SetFieldValue uses a column of columns to set a multi-column value.
func (v *View) SetFieldValue(columnID uint64, columnDepth uint, value uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
}
return frag.SetFieldValue(columnID, bitDepth, value)
return frag.SetFieldValue(columnID, columnDepth, value)
}
// FieldSum returns the sum & count of a field.
func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
func (v *View) FieldSum(filter *Row, columnDepth uint) (sum, count uint64, err error) {
for _, f := range v.Fragments() {
fsum, fcount, err := f.FieldSum(filter, bitDepth)
fsum, fcount, err := f.FieldSum(filter, columnDepth)
if err != nil {
return sum, count, err
}
@ -359,10 +359,10 @@ func (v *View) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err erro
}
// FieldMin returns the min and count of a field.
func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err error) {
func (v *View) FieldMin(filter *Row, columnDepth uint) (min, count uint64, err error) {
var minHasValue bool
for _, f := range v.Fragments() {
fmin, fcount, err := f.FieldMin(filter, bitDepth)
fmin, fcount, err := f.FieldMin(filter, columnDepth)
if err != nil {
return min, count, err
}
@ -387,9 +387,9 @@ func (v *View) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err erro
}
// FieldMax returns the max and count of a field.
func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err error) {
func (v *View) FieldMax(filter *Row, columnDepth uint) (max, count uint64, err error) {
for _, f := range v.Fragments() {
fmax, fcount, err := f.FieldMax(filter, bitDepth)
fmax, fcount, err := f.FieldMax(filter, columnDepth)
if err != nil {
return max, count, err
}
@ -402,10 +402,10 @@ func (v *View) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err erro
}
// FieldRange returns rows with a field value encoding matching the predicate.
func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row, error) {
func (v *View) FieldRange(op pql.Token, columnDepth uint, predicate uint64) (*Row, error) {
r := NewRow()
for _, frag := range v.Fragments() {
other, err := frag.FieldRange(op, bitDepth, predicate)
other, err := frag.FieldRange(op, columnDepth, predicate)
if err != nil {
return nil, err
}
@ -416,10 +416,10 @@ func (v *View) FieldRange(op pql.Token, bitDepth uint, predicate uint64) (*Row,
// FieldRangeBetween returns bitmaps with a field value encoding matching any
// value between predicateMin and predicateMax.
func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
func (v *View) FieldRangeBetween(columnDepth uint, predicateMin, predicateMax uint64) (*Row, error) {
r := NewRow()
for _, frag := range v.Fragments() {
other, err := frag.FieldRangeBetween(bitDepth, predicateMin, predicateMax)
other, err := frag.FieldRangeBetween(columnDepth, predicateMin, predicateMax)
if err != nil {
return nil, err
}

8
view_test.go
View file

@ -72,9 +72,9 @@ func (v *View) Reopen() error {
return v.Open()
}
// MustSetBits sets bits on a row. Panic on error.
// MustSetColumns sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) {
func (v *View) MustSetColumns(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := v.SetBit(rowID, columnID); err != nil {
panic(err)
@ -82,8 +82,8 @@ func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) {
}
}
// MustClearBits clears bits on a row. Panic on error.
func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) {
// MustClearColumns clears columns on a row. Panic on error.
func (v *View) MustClearColumns(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := v.ClearBit(rowID, columnID); err != nil {
panic(err)