From d34687121f4100986b5d68c6aad3f87826e10fa0 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Fri, 21 Apr 2017 15:59:41 -0600 Subject: [PATCH] Rename bitmap/profile to row/column. --- NOTES | 4 +- README.md | 14 +- attr.go | 2 +- attr_test.go | 6 +- cache.go | 64 ++++----- client.go | 50 +++---- client_test.go | 38 +++--- cmd/export.go | 2 +- cmd/import.go | 2 +- cmd/sort.go | 2 +- ctl/bench.go | 10 +- ctl/import.go | 16 +-- ctl/sort.go | 26 ++-- db.go | 22 ++-- executor.go | 76 +++++------ executor_test.go | 66 +++++----- fragment.go | 292 ++++++++++++++++++++--------------------- fragment_test.go | 171 ++++++++++-------------- frame.go | 58 ++++---- frame_test.go | 4 +- handler.go | 88 ++++++------- handler_test.go | 50 +++---- index.go | 22 ++-- index_test.go | 16 +-- internal/private.pb.go | 132 +++++++++---------- internal/private.proto | 6 +- internal/public.pb.go | 215 +++++++++++++++--------------- internal/public.proto | 14 +- iterator.go | 104 +++++++-------- pilosa.go | 44 +++---- server/server_test.go | 126 +++++++++--------- view.go | 20 +-- view_test.go | 28 ++-- 33 files changed, 880 insertions(+), 910 deletions(-) diff --git a/NOTES b/NOTES index 49d372099..6ba447da0 100644 --- a/NOTES +++ b/NOTES @@ -1,10 +1,10 @@ - DB Profile + DB Column ┌───────────▼────────────────────────────┐ │0000000000000000000000000000000000000000│ │0000000000000000000000000000000000000000│ │0000000000000000000000000000000000000000│ -Bitmap──▶0000000000000000000000000000000000000000│ + Row──▶0000000000000000000000000000000000000000│ │0000000000000000000000000000000000000000│ │────────────────────────────────────────┤ │0000000000000000000000000000000000000000│ diff --git a/README.md b/README.md index 9943bf15f..d8318820b 100644 --- a/README.md +++ b/README.md @@ -209,17 +209,17 @@ A return value of `{"results":[true]}` indicates that the bit was toggled from 1 A return value of `{"results":[false]}` indicates that the bit was already set to 0 and therefore nothing changed. --- -#### SetBitmapAttrs() +#### SetRowAttrs() ``` -SetBitmapAttrs(project=10, frame="collaboration", stars=123, url="http://projects.pilosa.com/10", active=true) +SetRowAttrs(project=10, frame="collaboration", stars=123, url="http://projects.pilosa.com/10", active=true) ``` Returns `{"results":[null]}` --- -#### SetProfileAttrs() +#### SetColumnAttrs() --- ``` -SetProfileAttrs(user=10, friends=123, username="mrpi", active=true) +SetColumnAttrs(user=10, friends=123, username="mrpi", active=true) ``` Returns `{"results":[null]}` @@ -230,11 +230,11 @@ Returns `{"results":[null]}` Bitmap(project=10, frame="collaboration") ``` Returns `{"results":[{"attrs":{"stars":123, "url":"http://projects.pilosa.com/10", "active":true},"bits":[1,2]}]}` where `attrs` are the -attributes set using `SetBitmapAttrs()` and `bits` are the bits set using `SetBit()`. +attributes set using `SetRowAttrs()` and `bits` are the bits set using `SetBit()`. -In order to return profile attributes attached to the profiles of a bitmap, add `&profiles=true` to the query string. Sample response: +In order to return column attributes attached to the columns of a bitmap, add `&columnAttrs=true` to the query string. Sample response: ``` -{"results":[{"attrs":{},"bits":[10]}],"profiles":[{"user":10,"attrs":{"friends":123, "username":"mrpi", "active":true}}]} +{"results":[{"attrs":{},"bits":[10]}],"columnAttrs":[{"user":10,"attrs":{"friends":123, "username":"mrpi", "active":true}}]} ``` --- diff --git a/attr.go b/attr.go index f619dda9f..3859a1dc7 100644 --- a/attr.go +++ b/attr.go @@ -226,7 +226,7 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro return m, nil } -// txAttrs returns a map of attributes for a bitmap. +// txAttrs returns a map of attributes for an id. func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) { v := tx.Bucket([]byte("attrs")).Get(u64tob(id)) if v == nil { diff --git a/attr_test.go b/attr_test.go index f026f6244..cfc31d54b 100644 --- a/attr_test.go +++ b/attr_test.go @@ -9,7 +9,7 @@ import ( "github.com/pilosa/pilosa" ) -// Ensure database can set and retrieve profile attributes. +// Ensure database can set and retrieve column attributes. func TestAttrStore_Attrs(t *testing.T) { s := MustOpenAttrStore() defer s.Close() @@ -23,14 +23,14 @@ func TestAttrStore_Attrs(t *testing.T) { t.Fatal(err) } - // Retrieve attributes for profile #1. + // Retrieve attributes for column #1. if m, err := s.Attrs(1); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) { t.Fatalf("unexpected attrs(1): %#v", m) } - // Retrieve attributes for profile #2. + // Retrieve attributes for column #2. if m, err := s.Attrs(2); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) { diff --git a/cache.go b/cache.go index 7da28eb74..1c338302e 100644 --- a/cache.go +++ b/cache.go @@ -17,15 +17,15 @@ const ( ThresholdFactor = 1.1 ) -// Cache represents a cache for bitmap counts. +// Cache represents a cache of counts. type Cache interface { - Add(bitmapID uint64, n uint64) - BulkAdd(bitmapID uint64, n uint64) - Get(bitmapID uint64) uint64 + Add(id uint64, n uint64) + BulkAdd(id uint64, n uint64) + Get(id uint64) uint64 Len() int - // Returns a list of all bitmap IDs. - BitmapIDs() []uint64 + // Returns a list of all IDs. + IDs() []uint64 // Updates the cache, if necessary. Invalidate() @@ -53,19 +53,19 @@ func NewLRUCache(maxEntries uint32) *LRUCache { return c } -func (c *LRUCache) BulkAdd(bitmapID, n uint64) { - c.Add(bitmapID, n) +func (c *LRUCache) BulkAdd(id, n uint64) { + c.Add(id, n) } -// Add adds a bitmap to the cache. -func (c *LRUCache) Add(bitmapID, n uint64) { - c.cache.Add(bitmapID, n) - c.counts[bitmapID] = n +// Add adds a count to the cache. +func (c *LRUCache) Add(id, n uint64) { + c.cache.Add(id, n) + c.counts[id] = n } -// Get returns a bitmap with a given id. -func (c *LRUCache) Get(bitmapID uint64) uint64 { - n, _ := c.cache.Get(bitmapID) +// Get returns a count for a given id. +func (c *LRUCache) Get(id uint64) uint64 { + n, _ := c.cache.Get(id) nn, _ := n.(uint64) return nn } @@ -79,8 +79,8 @@ func (c *LRUCache) Invalidate() {} // Recalculate is a no-op. func (c *LRUCache) Recalculate() {} -// BitmapIDs returns a list of all bitmap IDs in the cache. -func (c *LRUCache) BitmapIDs() []uint64 { +// IDs returns a list of all IDs in the cache. +func (c *LRUCache) IDs() []uint64 { a := make([]uint64, 0, len(c.counts)) for id := range c.counts { a = append(a, id) @@ -136,36 +136,36 @@ func NewRankCache(maxEntries uint32) *RankCache { } } -// Add adds a bitmap to the cache. -func (c *RankCache) Add(bitmapID uint64, n uint64) { +// Add adds a count to the cache. +func (c *RankCache) Add(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() - // Ignore if the bit count on the bitmap is below the threshold. + // Ignore if the bit count is below the threshold. if n < c.thresholdValue { return } - c.entries[bitmapID] = n + c.entries[id] = n c.invalidate() } -// BulkAdd adds a bitmap to the cache unsorted. You should Invalidate after completion. -func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) { +// BulkAdd adds a count to the cache unsorted. You should Invalidate after completion. +func (c *RankCache) BulkAdd(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() if n < c.thresholdValue { return } - c.entries[bitmapID] = n + c.entries[id] = n } -// Get returns a bitmap with a given id. -func (c *RankCache) Get(bitmapID uint64) uint64 { +// Get returns a count for a given id. +func (c *RankCache) Get(id uint64) uint64 { c.mu.Lock() defer c.mu.Unlock() - return c.entries[bitmapID] + return c.entries[id] } // Len returns the number of items in the cache. @@ -175,8 +175,8 @@ func (c *RankCache) Len() int { return len(c.entries) } -// BitmapIDs returns a list of all bitmap IDs in the cache. -func (c *RankCache) BitmapIDs() []uint64 { +// IDs returns a list of all IDs in the cache. +func (c *RankCache) IDs() []uint64 { c.mu.Lock() defer c.mu.Unlock() a := make([]uint64, 0, len(c.entries)) @@ -242,7 +242,7 @@ func (c *RankCache) recalculate() { } } -// Top returns an ordered list of bitmaps. +// Top returns an ordered list of pairs. func (c *RankCache) Top() []BitmapPair { return c.rankings } // WriteTo writes the cache to w. @@ -258,7 +258,7 @@ func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) { // Ensure RankCache implements Cache. var _ Cache = &RankCache{} -// BitmapPair represents a bitmap with an associated identifier. +// BitmapPair represents a id/count pair with an associated identifier. type BitmapPair struct { ID uint64 Count uint64 @@ -271,7 +271,7 @@ func (p BitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitmapPairs) Len() int { return len(p) } func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count } -// Pair holds a bitmap id and its count. +// Pair holds an id/count pair. type Pair struct { ID uint64 `json:"id"` Count uint64 `json:"count"` diff --git a/client.go b/client.go index e4401a69a..46a27c02f 100644 --- a/client.go +++ b/client.go @@ -316,9 +316,9 @@ func (c *Client) Import(ctx context.Context, db, frame string, slice uint64, bit } func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, error) { - // Separate bitmap and profile IDs to reduce allocations. - bitmapIDs := Bits(bits).BitmapIDs() - profileIDs := Bits(bits).ProfileIDs() + // Separate row and column IDs to reduce allocations. + rowIDs := Bits(bits).RowIDs() + columnIDs := Bits(bits).ColumnIDs() timestamps := Bits(bits).Timestamps() // Marshal bits to protobufs. @@ -326,8 +326,8 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e DB: db, Frame: frame, Slice: slice, - BitmapIDs: bitmapIDs, - ProfileIDs: profileIDs, + RowIDs: rowIDs, + ColumnIDs: columnIDs, Timestamps: timestamps, }) if err != nil { @@ -823,7 +823,7 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, sli return rsp.Blocks, nil } -// BlockData returns bitmap/profile id pairs for a block. +// BlockData returns row/column id pairs for a block. func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) { buf, err := proto.Marshal(&internal.BlockDataRequest{ DB: db, @@ -867,11 +867,11 @@ func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice ui } else if err := proto.Unmarshal(body, &rsp); err != nil { return nil, nil, err } - return rsp.BitmapIDs, rsp.ProfileIDs, nil + return rsp.RowIDs, rsp.ColumnIDs, nil } -// ProfileAttrDiff returns data from differing blocks on a remote host. -func (c *Client) ProfileAttrDiff(ctx context.Context, db string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +// ColumnAttrDiff returns data from differing blocks on a remote host. +func (c *Client) ColumnAttrDiff(ctx context.Context, db string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { u := url.URL{ Scheme: "http", Host: c.host, @@ -913,8 +913,8 @@ func (c *Client) ProfileAttrDiff(ctx context.Context, db string, blks []AttrBloc return rsp.Attrs, nil } -// BitmapAttrDiff returns data from differing blocks on a remote host. -func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { +// RowAttrDiff returns data from differing blocks on a remote host. +func (c *Client) RowAttrDiff(ctx context.Context, db, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) { u := url.URL{ Scheme: "http", Host: c.host, @@ -960,8 +960,8 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At // Bit represents the location of a single bit. type Bit struct { - BitmapID uint64 - ProfileID uint64 + RowID uint64 + ColumnID uint64 Timestamp int64 } @@ -972,29 +972,29 @@ func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p Bits) Len() int { return len(p) } func (p Bits) Less(i, j int) bool { - if p[i].BitmapID == p[j].BitmapID { - if p[i].ProfileID < p[j].ProfileID { + if p[i].RowID == p[j].RowID { + if p[i].ColumnID < p[j].ColumnID { return p[i].Timestamp < p[j].Timestamp } - return p[i].ProfileID < p[j].ProfileID + return p[i].ColumnID < p[j].ColumnID } - return p[i].BitmapID < p[j].BitmapID + return p[i].RowID < p[j].RowID } -// BitmapIDs returns a slice of all the bitmap IDs. -func (a Bits) BitmapIDs() []uint64 { +// RowIDs returns a slice of all the row IDs. +func (a Bits) RowIDs() []uint64 { other := make([]uint64, len(a)) for i := range a { - other[i] = a[i].BitmapID + other[i] = a[i].RowID } return other } -// ProfileIDs returns a slice of all the profile IDs. -func (a Bits) ProfileIDs() []uint64 { +// ColumnIDs returns a slice of all the column IDs. +func (a Bits) ColumnIDs() []uint64 { other := make([]uint64, len(a)) for i := range a { - other[i] = a[i].ProfileID + other[i] = a[i].ColumnID } return other } @@ -1012,7 +1012,7 @@ func (a Bits) Timestamps() []int64 { func (a Bits) GroupBySlice() map[uint64][]Bit { m := make(map[uint64][]Bit) for _, bit := range a { - slice := bit.ProfileID / SliceWidth + slice := bit.ColumnID / SliceWidth m[slice] = append(m[slice], bit) } @@ -1030,7 +1030,7 @@ type BitsByPos []Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } func (p BitsByPos) Len() int { return len(p) } func (p BitsByPos) Less(i, j int) bool { - p0, p1 := Pos(p[i].BitmapID, p[i].ProfileID), Pos(p[j].BitmapID, p[j].ProfileID) + p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID) if p0 == p1 { return p[i].Timestamp < p[j].Timestamp } diff --git a/client_test.go b/client_test.go index f78ee69c4..a05032d57 100644 --- a/client_test.go +++ b/client_test.go @@ -28,7 +28,7 @@ func createCluster(c *pilosa.Cluster) ([]*Server, []*Index) { return server, idx } -// Test distributed TopN Bitmap count across 3 nodes. +// Test distributed TopN Row count across 3 nodes. func TestClient_MultiNode(t *testing.T) { cluster := NewCluster(3) s, idx := createCluster(cluster) @@ -162,7 +162,7 @@ func TestClient_Import(t *testing.T) { // Load bitmap into cache to ensure cache gets updated. f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) - f.Bitmap(0) + f.Row(0) s := NewServer() defer s.Close() @@ -174,18 +174,18 @@ func TestClient_Import(t *testing.T) { // Send import request. c := MustNewClient(s.Host()) if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{ - {BitmapID: 0, ProfileID: 1}, - {BitmapID: 0, ProfileID: 5}, - {BitmapID: 200, ProfileID: 6}, + {RowID: 0, ColumnID: 1}, + {RowID: 0, ColumnID: 5}, + {RowID: 200, ColumnID: 6}, }); err != nil { t.Fatal(err) } // Verify data. - if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) { + if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) { t.Fatalf("unexpected bits: %+v", a) } - if a := f.Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) { + if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) { t.Fatalf("unexpected bits: %+v", a) } } @@ -213,7 +213,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) { } // Load bitmap into cache to ensure cache gets updated. - f.Bitmap(0) + f.Row(0) s := NewServer() defer s.Close() @@ -225,22 +225,22 @@ func TestClient_ImportInverseEnabled(t *testing.T) { // Send import request. c := MustNewClient(s.Host()) if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{ - {BitmapID: 0, ProfileID: 1}, - {BitmapID: 0, ProfileID: 5}, - {BitmapID: 200, ProfileID: 5}, - {BitmapID: 200, ProfileID: 6}, + {RowID: 0, ColumnID: 1}, + {RowID: 0, ColumnID: 5}, + {RowID: 200, ColumnID: 5}, + {RowID: 200, ColumnID: 6}, }); err != nil { t.Fatal(err) } // Verify data. - if a := f.Bitmap(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) { + if a := f.Row(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) { t.Fatalf("unexpected bits: %+v", a) } - if a := f.Bitmap(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) { + if a := f.Row(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) { t.Fatalf("unexpected bits: %+v", a) } - if a := f.Bitmap(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) { + if a := f.Row(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) { t.Fatalf("unexpected bits: %+v", a) } } @@ -279,16 +279,16 @@ func TestClient_BackupRestore(t *testing.T) { } // Verify data. - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { t.Fatalf("unexpected bits(0): %+v", a) } - if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { + if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { t.Fatalf("unexpected bits: %+v", a) } } diff --git a/cmd/export.go b/cmd/export.go index fb22ee369..e4511d40f 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -23,7 +23,7 @@ the output is written to STDOUT. The format of the CSV file is: - BITMAPID,PROFILEID + ROWID,COLUMNID The file does not contain any headers. `, diff --git a/cmd/import.go b/cmd/import.go index b7998f750..d8bc932e8 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -21,7 +21,7 @@ of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: - BITMAPID,PROFILEID,[TIME] + ROWID,COLUMNID,[TIME] The file should contain no headers. The TIME column is optional and can be omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. diff --git a/cmd/sort.go b/cmd/sort.go index 0ea7d9007..668206070 100644 --- a/cmd/sort.go +++ b/cmd/sort.go @@ -25,7 +25,7 @@ Sorts the import data at PATH into the optimal sort order for importing. The format of the CSV file is: - BITMAPID,PROFILEID + ROWID,COLUMNID The file should contain no headers. `, diff --git a/ctl/bench.go b/ctl/bench.go index 2216bace0..167017f2f 100644 --- a/ctl/bench.go +++ b/ctl/bench.go @@ -63,17 +63,17 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e return pilosa.ErrFrameRequired } - const maxBitmapID = 1000 - const maxProfileID = 100000 + const maxRowID = 1000 + const maxColumnID = 100000 startTime := time.Now() // Execute operation continuously. for i := 0; i < cmd.N; i++ { - bitmapID := rand.Intn(maxBitmapID) - profileID := rand.Intn(maxProfileID) + rowID := rand.Intn(maxRowID) + columnID := rand.Intn(maxColumnID) - q := fmt.Sprintf(`SetBit(id=%d, frame="%s", profileID=%d)`, bitmapID, cmd.Frame, profileID) + q := fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID) if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil { return err diff --git a/ctl/import.go b/ctl/import.go index df8fb00eb..1ba8677cb 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -123,19 +123,19 @@ func (cmd *ImportCommand) importPath(ctx context.Context, path string) error { var bit pilosa.Bit - // Parse bitmap id. - bitmapID, err := strconv.ParseUint(record[0], 10, 64) + // Parse row id. + rowID, err := strconv.ParseUint(record[0], 10, 64) if err != nil { - return fmt.Errorf("invalid bitmap id on row %d: %q", rnum, record[0]) + return fmt.Errorf("invalid row id on row %d: %q", rnum, record[0]) } - bit.BitmapID = bitmapID + bit.RowID = rowID - // Parse bitmap id. - profileID, err := strconv.ParseUint(record[1], 10, 64) + // Parse column id. + columnID, err := strconv.ParseUint(record[1], 10, 64) if err != nil { - return fmt.Errorf("invalid profile id on row %d: %q", rnum, record[1]) + return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1]) } - bit.ProfileID = profileID + bit.ColumnID = columnID // Parse time, if exists. if len(record) > 2 && record[2] != "" { diff --git a/ctl/sort.go b/ctl/sort.go index 9d4b14277..5590d2538 100644 --- a/ctl/sort.go +++ b/ctl/sort.go @@ -45,7 +45,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error { r.FieldsPerRecord = -1 a := make([]pilosa.Bit, 0, 1000000) for { - bitmapID, profileID, timestamp, err := readCSVRow(r) + rowID, columnID, timestamp, err := readCSVRow(r) if err == io.EOF { break } else if err == errBlank { @@ -53,7 +53,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error { } else if err != nil { return err } - a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp}) + a = append(a, pilosa.Bit{RowID: rowID, ColumnID: columnID, Timestamp: timestamp}) } // Sort bits by position. @@ -65,10 +65,10 @@ func (cmd *SortCommand) Run(ctx context.Context) error { for _, bit := range a { // Write CSV to buffer. buf = buf[:0] - buf = strconv.AppendUint(buf, bit.BitmapID, 10) + buf = strconv.AppendUint(buf, bit.RowID, 10) buf = append(buf, ',') - buf = strconv.AppendUint(buf, bit.ProfileID, 10) + buf = strconv.AppendUint(buf, bit.ColumnID, 10) if bit.Timestamp != 0 { buf = append(buf, ',') @@ -91,8 +91,8 @@ func (cmd *SortCommand) Run(ctx context.Context) error { return nil } -// readCSVRow reads a bitmap/profile pair from a CSV row. -func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) { +// readCSVRow reads a row/column pair from a CSV row. +func readCSVRow(r *csv.Reader) (rowID, columnID uint64, timestamp int64, err error) { // Read CSV row. record, err := r.Read() if err != nil { @@ -106,16 +106,16 @@ func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record)) } - // Parse bitmap id. - bitmapID, err = strconv.ParseUint(record[0], 10, 64) + // Parse row id. + rowID, err = strconv.ParseUint(record[0], 10, 64) if err != nil { - return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0]) + return 0, 0, 0, fmt.Errorf("invalid row id: %q", record[0]) } - // Parse bitmap id. - profileID, err = strconv.ParseUint(record[1], 10, 64) + // Parse column id. + columnID, err = strconv.ParseUint(record[1], 10, 64) if err != nil { - return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1]) + return 0, 0, 0, fmt.Errorf("invalid column id: %q", record[1]) } // Parse timestamp, if available. @@ -127,7 +127,7 @@ func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err timestamp = t.UnixNano() } - return bitmapID, profileID, timestamp, nil + return rowID, columnID, timestamp, nil } // errBlank indicates a blank row in a CSV file. diff --git a/db.go b/db.go index 8bf33300e..049e81261 100644 --- a/db.go +++ b/db.go @@ -17,7 +17,7 @@ import ( // Default database settings. const ( - DefaultColumnLabel = "profileID" + DefaultColumnLabel = "columnID" ) // DB represents a container for frames. @@ -40,8 +40,8 @@ type DB struct { remoteMaxSlice uint64 remoteMaxInverseSlice uint64 - // Profile attribute storage and cache - profileAttrStore *AttrStore + // Column attribute storage and cache + columnAttrStore *AttrStore broadcaster Broadcaster stats StatsClient @@ -64,7 +64,7 @@ func NewDB(path, name string) (*DB, error) { remoteMaxSlice: 0, remoteMaxInverseSlice: 0, - profileAttrStore: NewAttrStore(filepath.Join(path, ".data")), + columnAttrStore: NewAttrStore(filepath.Join(path, ".data")), columnLabel: DefaultColumnLabel, @@ -79,8 +79,8 @@ func (db *DB) Name() string { return db.name } // Path returns the path the database was initialized with. func (db *DB) Path() string { return db.path } -// ProfileAttrStore returns the storage for profile attributes. -func (db *DB) ProfileAttrStore() *AttrStore { return db.profileAttrStore } +// ColumnAttrStore returns the storage for column attributes. +func (db *DB) ColumnAttrStore() *AttrStore { return db.columnAttrStore } // SetColumnLabel sets the column label. Persists to meta file on update. func (db *DB) SetColumnLabel(v string) error { @@ -131,7 +131,7 @@ func (db *DB) Open() error { return err } - if err := db.profileAttrStore.Open(); err != nil { + if err := db.columnAttrStore.Open(); err != nil { return err } @@ -220,8 +220,8 @@ func (db *DB) Close() error { defer db.mu.Unlock() // Close the attribute store. - if db.profileAttrStore != nil { - db.profileAttrStore.Close() + if db.columnAttrStore != nil { + db.columnAttrStore.Close() } // Close all frames. @@ -560,6 +560,6 @@ type importKey struct { } type importData struct { - BitmapIDs []uint64 - ProfileIDs []uint64 + RowIDs []uint64 + ColumnIDs []uint64 } diff --git a/executor.go b/executor.go index 9c156ec56..6bf3395e6 100644 --- a/executor.go +++ b/executor.go @@ -21,7 +21,7 @@ const ( DefaultFrame = "general" // MinThreshold is the lowest count to use in a Top-N operation when - // looking for additional bitmap/count pairs. + // looking for additional id/count pairs. MinThreshold = 1 ) @@ -71,8 +71,8 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices } // Optimize handling for bulk attribute insertion. - if hasOnlySetBitmapAttrs(q.Calls) { - return e.executeBulkSetBitmapAttrs(ctx, db, q.Calls, opt) + if hasOnlySetRowAttrs(q.Calls) { + return e.executeBulkSetRowAttrs(ctx, db, q.Calls, opt) } // Execute each call serially. @@ -102,10 +102,10 @@ func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slic return e.executeCount(ctx, db, c, slices, opt) case "SetBit": return e.executeSetBit(ctx, db, c, opt) - case "SetBitmapAttrs": - return nil, e.executeSetBitmapAttrs(ctx, db, c, opt) - case "SetProfileAttrs": - return nil, e.executeSetProfileAttrs(ctx, db, c, opt) + case "SetRowAttrs": + return nil, e.executeSetRowAttrs(ctx, db, c, opt) + case "SetColumnAttrs": + return nil, e.executeSetColumnAttrs(ctx, db, c, opt) case "TopN": return e.executeTopN(ctx, db, c, slices, opt) default: @@ -155,7 +155,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call } // Attach attributes for Bitmap() calls. - // If the column label is used then return profile attributes. + // If the column label is used then return column attributes. // If the row label is used then return bitmap attributes. bm, _ := other.(*Bitmap) if c.Name == "Bitmap" { @@ -164,7 +164,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call if d != nil { columnLabel := d.ColumnLabel() if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil { - attrs, err := d.ProfileAttrStore().Attrs(columnID) + attrs, err := d.ColumnAttrStore().Attrs(columnID) if err != nil { return nil, err } @@ -179,7 +179,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call if err != nil { return nil, err } - attrs, err := fr.BitmapAttrStore().Attrs(rowID) + attrs, err := fr.RowAttrStore().Attrs(rowID) if err != nil { return nil, err } @@ -214,7 +214,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql // This first performs the TopN() to determine the top results and then // requeries to retrieve the full counts for each of the top results. func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) { - bitmapIDs, _, err := c.UintSliceArg("ids") + rowIDs, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopN: %v", err) } @@ -231,7 +231,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic // If this call is against specific ids, or we didn't get results, // or we are part of a larger distributed query then don't refetch. - if len(pairs) == 0 || len(bitmapIDs) > 0 || opt.Remote { + if len(pairs) == 0 || len(rowIDs) > 0 || opt.Remote { return pairs, nil } // Only the original caller should refetch the full counts. @@ -284,7 +284,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, return nil, fmt.Errorf("executeTopNSlice: %v", err) } field, _ := c.Args["field"].(string) - bitmapIDs, _, err := c.UintSliceArg("ids") + rowIDs, _, err := c.UintSliceArg("ids") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) } @@ -330,7 +330,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call, return f.Top(TopOptions{ N: int(n), Src: src, - BitmapIDs: bitmapIDs, + RowIDs: rowIDs, FilterField: field, FilterValues: filters, MinThreshold: minThreshold, @@ -404,7 +404,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal if frag == nil { return NewBitmap(), nil } - return frag.Bitmap(id), nil + return frag.Row(id), nil } // executeIntersectSlice executes a intersect() call for a local slice. @@ -483,7 +483,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call if f == nil { continue } - bm = bm.Union(f.Bitmap(rowID)) + bm = bm.Union(f.Row(rowID)) } return bm, nil } @@ -739,11 +739,11 @@ func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call return ret, nil } -// executeSetBitmapAttrs executes a SetBitmapAttrs() call. -func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { +// executeSetRowAttrs executes a SetRowAttrs() call. +func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { frameName, ok := c.Args["frame"].(string) if !ok { - return errors.New("SetBitmapAttrs() frame required") + return errors.New("SetRowAttrs() frame required") } // Retrieve frame. @@ -756,9 +756,9 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. // Parse labels. rowID, ok, err := c.UintArg(rowLabel) if err != nil { - return fmt.Errorf("reading SetBitmapAttrs() row: %v", err) + return fmt.Errorf("reading SetRowAttrs() row: %v", err) } else if !ok { - return fmt.Errorf("SetBitmapAttrs() row field '%v' required.", rowLabel) + return fmt.Errorf("SetRowAttrs() row field '%v' required.", rowLabel) } // Copy args and remove reserved fields. @@ -767,7 +767,7 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. delete(attrs, rowLabel) // Set attributes. - if err := frame.BitmapAttrStore().SetAttrs(rowID, attrs); err != nil { + if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { return err } @@ -796,14 +796,14 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql. return nil } -// executeBulkSetBitmapAttrs executes a set of SetBitmapAttrs() calls. -func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { +// executeBulkSetRowAttrs executes a set of SetRowAttrs() calls. +func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) { // Collect attributes by frame/id. m := make(map[string]map[uint64]map[string]interface{}) for _, c := range calls { frame, ok := c.Args["frame"].(string) if !ok { - return nil, errors.New("SetBitmapAttrs() frame required") + return nil, errors.New("SetRowAttrs() frame required") } // Retrieve frame. @@ -815,9 +815,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal rowID, ok, err := c.UintArg(rowLabel) if err != nil { - return nil, fmt.Errorf("reading SetBitmapAttrs() row: %v", rowLabel) + return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel) } else if !ok { - return nil, fmt.Errorf("SetBitmapAttrs row field '%v' required.", rowLabel) + return nil, fmt.Errorf("SetRowAttrs row field '%v' required.", rowLabel) } // Copy args and remove reserved fields. @@ -852,7 +852,7 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal } // Set attributes. - if err := frame.BitmapAttrStore().SetBulkAttrs(frameMap); err != nil { + if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil { return nil, err } } @@ -883,8 +883,8 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal return make([]interface{}, len(calls)), nil } -// executeSetProfileAttrs executes a SetProfileAttrs() call. -func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { +// executeSetColumnAttrs executes a SetColumnAttrs() call. +func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error { // Retrieve database. d := e.Index.DB(db) if d == nil { @@ -898,7 +898,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql columnLabel := d.columnLabel col, okCol, errCol := c.UintArg(columnLabel) if errCol != nil || !okCol { - return fmt.Errorf("reading SetProfileAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol) + return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol) } id = col colName = columnLabel @@ -911,7 +911,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql delete(attrs, colName) // Set attributes. - if err := d.ProfileAttrStore().SetAttrs(id, attrs); err != nil { + if err := d.ColumnAttrStore().SetAttrs(id, attrs); err != nil { return err } @@ -1011,8 +1011,8 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query v, err = pb.Results[i].Changed, nil case "ClearBit": v, err = pb.Results[i].Changed, nil - case "SetBitmapAttrs": - case "SetProfileAttrs": + case "SetRowAttrs": + case "SetColumnAttrs": default: v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil } @@ -1207,14 +1207,14 @@ func decodeError(s string) error { return errors.New(s) } -// hasOnlySetBitmapAttrs returns true if calls only contains SetBitmapAttrs() calls. -func hasOnlySetBitmapAttrs(calls []*pql.Call) bool { +// hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls. +func hasOnlySetRowAttrs(calls []*pql.Call) bool { if len(calls) == 0 { return false } for _, call := range calls { - if call.Name != "SetBitmapAttrs" { + if call.Name != "SetRowAttrs" { return false } } @@ -1227,7 +1227,7 @@ func needsSlices(calls []*pql.Call) bool { } for _, call := range calls { switch call.Name { - case "ClearBit", "SetBit", "SetBitmapAttrs", "SetProfileAttrs": + case "ClearBit", "SetBit", "SetRowAttrs", "SetColumnAttrs": continue case "Count", "TopN": return true diff --git a/executor_test.go b/executor_test.go index 7a0c95622..97a89f3ae 100644 --- a/executor_test.go +++ b/executor_test.go @@ -27,13 +27,13 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "d", MustParse(``+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } - if err := f.BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + if err := f.RowAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } @@ -58,17 +58,17 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { // Set bits. if _, err := e.Execute(context.Background(), "d", MustParse(``+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, 3)+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, SliceWidth+1)+ - fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 20, SliceWidth+1), + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+ + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+ + fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1), ), nil, nil); err != nil { t.Fatal(err) } - if err := db.ProfileAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { + if err := db.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } - if res, err := e.Execute(context.Background(), "d", MustParse(fmt.Sprintf(`Bitmap(profileID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "d", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { t.Fatal(err) } else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { t.Fatalf("unexpected bits: %+v", bits) @@ -195,11 +195,11 @@ func TestExecutor_Execute_SetBit(t *testing.T) { e := NewExecutor(idx.Index, NewCluster(1)) f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0) - if n := f.Bitmap(11).Count(); n != 0 { + if n := f.Row(11).Count(); n != 0 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if !res[0].(bool) { @@ -207,10 +207,10 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } - if n := f.Bitmap(11).Count(); n != 1 { + if n := f.Row(11).Count(); n != 1 { t.Fatalf("unexpected bitmap count: %d", n) } - if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil { + if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil { t.Fatal(err) } else { if res[0].(bool) { @@ -219,8 +219,8 @@ func TestExecutor_Execute_SetBit(t *testing.T) { } } -// Ensure a SetBitmapAttrs() query can be executed. -func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { +// Ensure a SetRowAttrs() query can be executed. +func TestExecutor_Execute_SetRowAttrs(t *testing.T) { idx := MustOpenIndex() defer idx.Close() @@ -235,21 +235,21 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) { // Set two fields on f/10. // Also set fields on other bitmaps and frames to test isolation. e := NewExecutor(idx.Index, NewCluster(1)) - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil { t.Fatal(err) } - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil { t.Fatal(err) } f := idx.Frame("d", "f") - if m, err := f.BitmapAttrStore().Attrs(10); err != nil { + if m, err := f.RowAttrStore().Attrs(10); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) { t.Fatalf("unexpected bitmap attr: %#v", m) @@ -261,7 +261,7 @@ func TestExecutor_Execute_TopN(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - // Set bits for bitmaps 0, 10, & 20 across two slices. + // Set bits for rows 0, 10, & 20 across two slices. idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) @@ -287,7 +287,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - // Set bits for bitmaps 0, 10, & 20 across two slices. + // Set bits for rows 0, 10, & 20 across two slices. idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2) @@ -345,7 +345,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { idx := MustOpenIndex() defer idx.Close() - // Set bits for bitmaps 0, 10, & 20 across two slices. + // Set bits for rows 0, 10, & 20 across two slices. idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) @@ -355,7 +355,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2) - // Create an intersecting bitmap. + // Create an intersecting row. idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth) idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1) idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2) @@ -382,7 +382,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { + if err := idx.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil { t.Fatal(err) } e := NewExecutor(idx.Index, NewCluster(1)) @@ -405,7 +405,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) { idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1) idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth) - if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { + if err := idx.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil { t.Fatal(err) } e := NewExecutor(idx.Index, NewCluster(1)) @@ -445,7 +445,7 @@ func TestExecutor_Execute_Range(t *testing.T) { f.MustSetBit(pilosa.ViewStandard, 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early f.MustSetBit(pilosa.ViewStandard, 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late - f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap + f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different row e := NewExecutor(idx.Index, NewCluster(1)) if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { @@ -542,7 +542,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) - } else if query.String() != `SetBit(frame="f", id=10, profileID=2)` { + } else if query.String() != `SetBit(columnID=2, frame="f", id=10)` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -559,12 +559,12 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } e := NewExecutor(idx.Index, c) - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil { t.Fatal(err) } // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Bitmap(10).Count(); n != 1 { + if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { @@ -587,7 +587,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) { if db != `d` { t.Fatalf("unexpected db: %s", db) - } else if query.String() != `SetBit(frame="f", id=10, profileID=2, timestamp="2016-12-11T10:09")` { + } else if query.String() != `SetBit(columnID=2, frame="f", id=10, timestamp="2016-12-11T10:09")` { t.Fatalf("unexpected query: %s", query.String()) } remoteCalled = true @@ -606,12 +606,12 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { } e := NewExecutor(idx.Index, c) - if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { + if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil { t.Fatal(err) } // Verify that one bit is set on both node's index. - if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Bitmap(10).Count(); n != 1 { + if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } if !remoteCalled { diff --git a/fragment.go b/fragment.go index 9bee4288e..9f68df30e 100644 --- a/fragment.go +++ b/fragment.go @@ -29,7 +29,7 @@ import ( ) const ( - // SliceWidth is the number of profile IDs in a slice. + // SliceWidth is the number of column IDs in a slice. SliceWidth = 1048576 // SnapshotExt is the file extension used for an in-process snapshot. @@ -41,7 +41,7 @@ const ( // CacheExt is the file extension for persisted cache ids. CacheExt = ".cache" - // HashBlockSize is the number of bitmaps in a merkle hash block. + // HashBlockSize is the number of rows in a merkle hash block. HashBlockSize = 100 ) @@ -67,13 +67,13 @@ type Fragment struct { storageData []byte opN int // number of ops since snapshot - // Cache for bitmap counts. + // Cache for row counts. cacheType string // passed in by frame cache Cache cacheSize uint32 - // Cache containing full bitmaps (not just counts). - bitmapCache BitmapCache + // Cache containing full rows (not just counts). + rowCache BitmapCache // Cached checksums for each block. checksums map[int][]byte @@ -86,9 +86,9 @@ type Fragment struct { // Writer used for out-of-band log entries. LogOutput io.Writer - // Bitmap attribute storage. + // Row attribute storage. // This is set by the parent frame unless overridden for testing. - BitmapAttrStore *AttrStore + RowAttrStore *AttrStore stats StatsClient } @@ -144,7 +144,7 @@ func (f *Fragment) Open() error { return err } - // Fill cache with bitmaps persisted to disk. + // Fill cache with rows persisted to disk. if err := f.openCache(); err != nil { return err } @@ -213,13 +213,13 @@ func (f *Fragment) openStorage() error { // Attach the file to the bitmap to act as a write-ahead log. f.storage.OpWriter = f.file - f.bitmapCache = &SimpleCache{make(map[uint64]*Bitmap)} + f.rowCache = &SimpleCache{make(map[uint64]*Bitmap)} return nil } -// openCache initializes the cache from bitmap ids persisted to disk. +// openCache initializes the cache from row ids persisted to disk. func (f *Fragment) openCache() error { // Determine cache type from frame name. switch f.cacheType { @@ -247,12 +247,12 @@ func (f *Fragment) openCache() error { return nil } - // Read in all bitmaps by ID. + // Read in all rows by ID. // This will cause them to be added to the cache. - for _, bitmapID := range pb.BitmapIDs { - //n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) - n := f.bitmap(bitmapID, true, true).Count() - f.cache.BulkAdd(bitmapID, n) + for _, id := range pb.IDs { + //n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth) + n := f.row(id, true, true).Count() + f.cache.BulkAdd(id, n) } f.cache.Invalidate() @@ -314,17 +314,16 @@ func (f *Fragment) closeStorage() error { // logger returns a logger instance for the fragment.nt. func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.LstdFlags) } -// Bitmap returns a bitmap by ID. -func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap { +// Row returns a row by ID. +func (f *Fragment) Row(rowID uint64) *Bitmap { f.mu.Lock() defer f.mu.Unlock() - return f.bitmap(bitmapID, true, true) + return f.row(rowID, true, true) } -func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCache bool) *Bitmap { - - if checkBitmapCache { - r, ok := f.bitmapCache.Fetch(bitmapID) +func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Bitmap { + if checkRowCache { + r, ok := f.rowCache.Fetch(rowID) if ok && r != nil { return r } @@ -332,11 +331,11 @@ func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCa // Only use a subset of the containers. // NOTE: The start & end ranges must be divisible by - data := f.storage.OffsetRange(f.slice*SliceWidth, bitmapID*SliceWidth, (bitmapID+1)*SliceWidth) + data := f.storage.OffsetRange(f.slice*SliceWidth, rowID*SliceWidth, (rowID+1)*SliceWidth) // Reference bitmap subrange in storage. // We Clone() data because otherwise bm will contains pointers to containers in storage. - // This causes unexpected results when we cache the bitmap and try to use it later. + // This causes unexpected results when we cache the row and try to use it later. bm := &Bitmap{ segments: []BitmapSegment{{ data: *data.Clone(), @@ -346,25 +345,25 @@ func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCa } bm.InvalidateCount() - if updateBitmapCache { - f.bitmapCache.Add(bitmapID, bm) + if updateRowCache { + f.rowCache.Add(rowID, bm) } return bm } -// SetBit sets a bit for a given profile & bitmap within the fragment. +// SetBit sets a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) { +func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) { f.mu.Lock() defer f.mu.Unlock() - return f.setBit(bitmapID, profileID) + return f.setBit(rowID, columnID) } -func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error) { +func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. - pos, err := f.pos(bitmapID, profileID) + pos, err := f.pos(rowID, columnID) if err != nil { return false, err } @@ -380,37 +379,37 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error) } // Invalidate block checksum. - delete(f.checksums, int(bitmapID/HashBlockSize)) + delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. if err := f.incrementOpN(); err != nil { return false, err } - // Get the bitmap from bitmapCache or fragment.storage. - bm := f.bitmap(bitmapID, true, true) - bm.SetBit(profileID) + // Get the row from row cache or fragment.storage. + bm := f.row(rowID, true, true) + bm.SetBit(columnID) // Update the cache. - f.cache.Add(bitmapID, bm.Count()) + f.cache.Add(rowID, bm.Count()) f.stats.Count("setN", 1) return changed, nil } -// ClearBit clears a bit for a given profile & bitmap within the fragment. +// ClearBit clears a bit for a given column & row within the fragment. // This updates both the on-disk storage and the in-cache bitmap. -func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) { +func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) { f.mu.Lock() defer f.mu.Unlock() - return f.clearBit(bitmapID, profileID) + return f.clearBit(rowID, columnID) } -func (f *Fragment) clearBit(bitmapID, profileID uint64) (changed bool, err error) { +func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) { changed = false // Determine the position of the bit in the storage. - pos, err := f.pos(bitmapID, profileID) + pos, err := f.pos(rowID, columnID) if err != nil { return false, err } @@ -426,38 +425,38 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (changed bool, err error } // Invalidate block checksum. - delete(f.checksums, int(bitmapID/HashBlockSize)) + delete(f.checksums, int(rowID/HashBlockSize)) // Increment number of operations until snapshot is required. if err := f.incrementOpN(); err != nil { return false, err } - // Get the bitmap from bitmapCache or fragment.storage. - bm := f.bitmap(bitmapID, true, true) - bm.ClearBit(profileID) + // Get the row from cache or fragment.storage. + bm := f.row(rowID, true, true) + bm.ClearBit(columnID) // Update the cache. - f.cache.Add(bitmapID, bm.Count()) + f.cache.Add(rowID, bm.Count()) f.stats.Count("clearN", 1) return changed, nil } -// pos translates the bitmap ID and profile ID into a position in the storage bitmap. -func (f *Fragment) pos(bitmapID, profileID uint64) (uint64, error) { - // Return an error if the profile ID is out of the range of the fragment's slice. - minProfileID := f.slice * SliceWidth - if profileID < minProfileID || profileID >= minProfileID+SliceWidth { - return 0, errors.New("profile out of bounds") +// pos translates the row ID and column ID into a position in the storage bitmap. +func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) { + // Return an error if the column ID is out of the range of the fragment's slice. + minColumnID := f.slice * SliceWidth + if columnID < minColumnID || columnID >= minColumnID+SliceWidth { + return 0, errors.New("column out of bounds") } - return Pos(bitmapID, profileID), nil + return Pos(rowID, columnID), nil } // ForEachBit executes fn for every bit set in the fragment. // Errors returned from fn are passed through. -func (f *Fragment) ForEachBit(fn func(bitmapID, profileID uint64) error) error { +func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error { f.mu.Lock() defer f.mu.Unlock() @@ -474,15 +473,15 @@ func (f *Fragment) ForEachBit(fn func(bitmapID, profileID uint64) error) error { return err } -// Top returns the top bitmaps from the fragment. -// If opt.Src is specified then only bitmaps which intersect src are returned. -// If opt.FilterValues exist then the bitmap attribute specified by field is matched. +// Top returns the top rows from the fragment. +// If opt.Src is specified then only rows which intersect src are returned. +// If opt.FilterValues exist then the row attribute specified by field is matched. func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { - // Retrieve pairs. If no bitmap ids specified then return from cache. - pairs := f.topBitmapPairs(opt.BitmapIDs) + // Retrieve pairs. If no row ids specified then return from cache. + pairs := f.topBitmapPairs(opt.RowIDs) - // If BitmapIDs are provided, we don't want to truncate the result set - if len(opt.BitmapIDs) > 0 { + // If row ids are provided, we don't want to truncate the result set + if len(opt.RowIDs) > 0 { opt.N = 0 } @@ -509,9 +508,9 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Iterate over rankings and add to results until we have enough. results := &PairHeap{} for _, pair := range pairs { - bitmapID, cnt := pair.ID, pair.Count + rowID, cnt := pair.ID, pair.Count - // Ignore empty bitmaps. + // Ignore empty rows. if cnt <= 0 { continue } @@ -531,7 +530,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Apply filter, if set. if filters != nil { - attr, err := f.BitmapAttrStore.Attrs(bitmapID) + attr, err := f.RowAttrStore.Attrs(rowID) if err != nil { return nil, err } else if attr == nil { @@ -548,7 +547,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // Calculate count and append. count := cnt if opt.Src != nil { - count = opt.Src.IntersectionCount(f.Bitmap(bitmapID)) + count = opt.Src.IntersectionCount(f.Row(rowID)) } if count == 0 { continue @@ -566,7 +565,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { } } - heap.Push(results, Pair{ID: bitmapID, Count: count}) + heap.Push(results, Pair{ID: rowID, Count: count}) // If we reach the requested number of pairs and we are not computing // intersections then simply exit. If we are intersecting then sort @@ -584,20 +583,20 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // If it's too low then don't try finding anymore pairs. threshold := results.Pairs[0].Count - // If the bitmap doesn't have enough bits set before the intersection - // then we can assume that any remaining bitmaps also have a count too low. + // If the row doesn't have enough bits set before the intersection + // then we can assume that any remaining rows also have a count too low. if threshold < opt.MinThreshold || cnt < threshold { break } // Calculate the intersecting bit count and skip if it's below our - // last bitmap in our current result set. - count := opt.Src.IntersectionCount(f.Bitmap(bitmapID)) + // last row in our current result set. + count := opt.Src.IntersectionCount(f.Row(rowID)) if count < threshold { continue } - heap.Push(results, Pair{ID: bitmapID, Count: count}) + heap.Push(results, Pair{ID: rowID, Count: count}) } //Pop first opt.N elements out of heap @@ -611,32 +610,32 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { return r, nil } -func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { - // If no specific bitmaps are requested, retrieve top bitmaps. - if len(bitmapIDs) == 0 { +func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair { + // If no specific rows are requested, retrieve top rows. + if len(rowIDs) == 0 { f.mu.Lock() defer f.mu.Unlock() f.cache.Invalidate() return f.cache.Top() } - // Otherwise retrieve specific bitmaps. - pairs := make([]BitmapPair, 0, len(bitmapIDs)) - for _, bitmapID := range bitmapIDs { + // Otherwise retrieve specific rows. + pairs := make([]BitmapPair, 0, len(rowIDs)) + for _, rowID := range rowIDs { // Look up cache first, if available. - if n := f.cache.Get(bitmapID); n > 0 { + if n := f.cache.Get(rowID); n > 0 { pairs = append(pairs, BitmapPair{ - ID: bitmapID, + ID: rowID, Count: n, }) continue } - bm := f.Bitmap(bitmapID) + bm := f.Row(rowID) if bm.Count() > 0 { // Otherwise load from storage. pairs = append(pairs, BitmapPair{ - ID: bitmapID, + ID: rowID, Count: bm.Count(), }) } @@ -647,14 +646,14 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair { // TopOptions represents options passed into the Top() function. type TopOptions struct { - // Number of bitmaps to return. + // Number of rows to return. N int // Bitmap to intersect with. Src *Bitmap - // Specific bitmaps to filter against. - BitmapIDs []uint64 + // Specific rows to filter against. + RowIDs []uint64 MinThreshold uint64 // Filter field name & values. @@ -768,14 +767,14 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i } } -// BlockData returns bits in a block as bitmap & profile ID pairs. -func (f *Fragment) BlockData(id int) (bitmapIDs, profileIDs []uint64) { +// BlockData returns bits in a block as row & column ID pairs. +func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) { f.mu.Lock() defer f.mu.Unlock() f.storage.ForEachRange(uint64(id)*HashBlockSize*SliceWidth, (uint64(id)+1)*HashBlockSize*SliceWidth, func(i uint64) { - bitmapIDs = append(bitmapIDs, i/SliceWidth) - profileIDs = append(profileIDs, i%SliceWidth) + rowIDs = append(rowIDs, i/SliceWidth) + columnIDs = append(columnIDs, i%SliceWidth) }) return } @@ -789,8 +788,8 @@ func (f *Fragment) BlockData(id int) (bitmapIDs, profileIDs []uint64) { func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) { // Ensure that all pair sets are of equal length. for i := range data { - if len(data[i].BitmapIDs) != len(data[i].ProfileIDs) { - return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].BitmapIDs), len(data[i].ProfileIDs)) + if len(data[i].RowIDs) != len(data[i].ColumnIDs) { + return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs)) } } @@ -801,22 +800,22 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e sets = make([]PairSet, len(data)+1) clears = make([]PairSet, len(data)+1) - // Limit upper bitmap/profile pair. - maxBitmapID := uint64(id+1) * HashBlockSize - maxProfileID := uint64(SliceWidth) + // Limit upper row/column pair. + maxRowID := uint64(id+1) * HashBlockSize + maxColumnID := uint64(SliceWidth) // Create buffered iterator for local block. itrs := make([]*BufIterator, 1, len(data)+1) itrs[0] = NewBufIterator( NewLimitIterator( - NewRoaringIterator(f.storage.Iterator()), maxBitmapID, maxProfileID, + NewRoaringIterator(f.storage.Iterator()), maxRowID, maxColumnID, ), ) // Append buffered iterators for each incoming block. for i := range data { - var itr Iterator = NewSliceIterator(data[i].BitmapIDs, data[i].ProfileIDs) - itr = NewLimitIterator(itr, maxBitmapID, maxProfileID) + var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs) + itr = NewLimitIterator(itr, maxRowID, maxColumnID) itrs = append(itrs, NewBufIterator(itr)) } @@ -833,8 +832,8 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e values := make([]bool, len(itrs)) for { var min struct { - bitmapID uint64 - profileID uint64 + rowID uint64 + columnID uint64 } // Find the lowest pair. @@ -844,9 +843,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e if eof { // no more data continue } else if !hasData { // first pair - min.bitmapID, min.profileID, hasData = bid, pid, true - } else if bid < min.bitmapID || (bid == min.bitmapID && pid < min.profileID) { // lower pair - min.bitmapID, min.profileID = bid, pid + min.rowID, min.columnID, hasData = bid, pid, true + } else if bid < min.rowID || (bid == min.rowID && pid < min.columnID) { // lower pair + min.rowID, min.columnID = bid, pid } } @@ -860,7 +859,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e for i, itr := range itrs { bid, pid, eof := itr.Next() - values[i] = !eof && bid == min.bitmapID && pid == min.profileID + values[i] = !eof && bid == min.rowID && pid == min.columnID if values[i] { setN++ // set } else { @@ -880,25 +879,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Append to either the set or clear diff. if newValue { - sets[i].BitmapIDs = append(sets[i].BitmapIDs, min.bitmapID) - sets[i].ProfileIDs = append(sets[i].ProfileIDs, min.profileID) + sets[i].RowIDs = append(sets[i].RowIDs, min.rowID) + sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) } else { - clears[i].BitmapIDs = append(sets[i].BitmapIDs, min.bitmapID) - clears[i].ProfileIDs = append(sets[i].ProfileIDs, min.profileID) + clears[i].RowIDs = append(sets[i].RowIDs, min.rowID) + clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID) } } } // Set local bits. - for i := range sets[0].ProfileIDs { - if _, err := f.setBit(sets[0].BitmapIDs[i], (f.Slice()*SliceWidth)+sets[0].ProfileIDs[i]); err != nil { + for i := range sets[0].ColumnIDs { + if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil { return nil, nil, err } } // Clear local bits. - for i := range clears[0].ProfileIDs { - if _, err := f.clearBit(clears[0].BitmapIDs[i], (f.Slice()*SliceWidth)+clears[0].ProfileIDs[i]); err != nil { + for i := range clears[0].ColumnIDs { + if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil { return nil, nil, err } } @@ -908,12 +907,12 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e // Import bulk imports a set of bits and then snapshots the storage. // This does not affect the fragment's cache. -func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { +func (f *Fragment) Import(rowIDs, columnIDs []uint64) error { f.mu.Lock() defer f.mu.Unlock() - // Verify that there are an equal number of bitmap ids and profile ids. - if len(bitmapIDs) != len(profileIDs) { - return fmt.Errorf("mismatch of bitmap/profile len: %d != %d", len(bitmapIDs), len(profileIDs)) + // Verify that there are an equal number of row ids and column ids. + if len(rowIDs) != len(columnIDs) { + return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs)) } // Disconnect op writer so we don't append updates. @@ -924,11 +923,11 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { lastID := uint64(0) if err := func() error { set := make(map[uint64]struct{}) - for i := range bitmapIDs { - bitmapID, profileID := bitmapIDs[i], profileIDs[i] + for i := range rowIDs { + rowID, columnID := rowIDs[i], columnIDs[i] // Determine the position of the bit in the storage. - pos, err := f.pos(bitmapID, profileID) + pos, err := f.pos(rowID, columnID) if err != nil { return err } @@ -942,21 +941,21 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error { // import optimization to avoid linear foreach calls // slight risk of concurrent cache counter being off but // no real danger - if i == 0 || bitmapID != lastID { - lastID = bitmapID - set[bitmapID] = struct{}{} + if i == 0 || rowID != lastID { + lastID = rowID + set[rowID] = struct{}{} } // Invalidate block checksum. - delete(f.checksums, int(bitmapID/HashBlockSize)) + delete(f.checksums, int(rowID/HashBlockSize)) } - // Update cache counts for all bitmaps. - for bitmapID := range set { - // Import should ALWAYS have bitmap() load a new bm from fragment.storage - // because the bitmap that's in bitmapCache hasn't been updated with + // Update cache counts for all rows. + for rowID := range set { + // Import should ALWAYS have row() load a new bm from fragment.storage + // because the row that's in rowCache hasn't been updated with // this import's data. - f.cache.BulkAdd(bitmapID, f.bitmap(bitmapID, false, false).Count()) + f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count()) } f.cache.Invalidate() @@ -995,6 +994,7 @@ func (f *Fragment) Snapshot() error { defer f.mu.Unlock() return f.snapshot() } + func track(start time.Time, name string, logger *log.Logger) { elapsed := time.Since(start) logger.Printf("%s took %s", name, elapsed) @@ -1061,13 +1061,11 @@ func (f *Fragment) flushCache() error { return nil } - // Retrieve a list of bitmap ids from the cache. - bitmapIDs := f.cache.BitmapIDs() + // Retrieve a list of row ids from the cache. + ids := f.cache.IDs() // Marshal cache data to bytes. - buf, err := proto.Marshal(&internal.Cache{ - BitmapIDs: bitmapIDs, - }) + buf, err := proto.Marshal(&internal.Cache{IDs: ids}) if err != nil { return err } @@ -1253,7 +1251,7 @@ func (f *Fragment) readCacheFromArchive(r io.Reader) error { return nil } -// FragmentBlock represents info about a subsection of the bitmaps in a block. +// FragmentBlock represents info about a subsection of the rows in a block. // This is used for comparing data in remote blocks for active anti-entropy. type FragmentBlock struct { ID int `json:"id"` @@ -1386,7 +1384,7 @@ func (s *FragmentSyncer) SyncFragment() error { return nil } -// syncBlock sends and receives all bitmaps for a given block. +// syncBlock sends and receives all rows for a given block. // Returns an error if any remote hosts are unreachable. func (s *FragmentSyncer) syncBlock(id int) error { f := s.Fragment @@ -1411,14 +1409,14 @@ func (s *FragmentSyncer) syncBlock(id int) error { clients = append(clients, client) // Only sync the standard block. - bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id) + rowIDs, columnIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id) if err != nil { return err } pairSets = append(pairSets, PairSet{ - ProfileIDs: profileIDs, - BitmapIDs: bitmapIDs, + ColumnIDs: columnIDs, + RowIDs: rowIDs, }) } @@ -1438,7 +1436,7 @@ func (s *FragmentSyncer) syncBlock(id int) error { set, clear := sets[i], clears[i] // Ignore if there are no differences. - if len(set.ProfileIDs) == 0 && len(clear.ProfileIDs) == 0 { + if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 { continue } @@ -1446,11 +1444,11 @@ func (s *FragmentSyncer) syncBlock(id int) error { var buf bytes.Buffer // Only sync the standard block. - for j := 0; j < len(set.ProfileIDs); j++ { - fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), set.BitmapIDs[j], (f.Slice()*SliceWidth)+set.ProfileIDs[j]) + for j := 0; j < len(set.ColumnIDs); j++ { + fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j]) } - for j := 0; j < len(clear.ProfileIDs); j++ { - fmt.Fprintf(&buf, "ClearBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), clear.BitmapIDs[j], (f.Slice()*SliceWidth)+clear.ProfileIDs[j]) + for j := 0; j < len(clear.ColumnIDs); j++ { + fmt.Fprintf(&buf, "ClearBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j]) } // Verify sync is not prematurely closing. @@ -1476,10 +1474,10 @@ func madvise(b []byte, advice int) (err error) { return } -// PairSet is a list of equal length bitmap and profile id lists. +// PairSet is a list of equal length row and column id lists. type PairSet struct { - BitmapIDs []uint64 - ProfileIDs []uint64 + RowIDs []uint64 + ColumnIDs []uint64 } // byteSlicesEqual returns true if all slices are equal. @@ -1496,7 +1494,7 @@ func byteSlicesEqual(a [][]byte) bool { return true } -// Pos returns the bitmap position of a bitmap/profile pair. -func Pos(bitmapID, profileID uint64) uint64 { - return (bitmapID * SliceWidth) + (profileID % SliceWidth) +// Pos returns the row position of a row/column pair. +func Pos(rowID, columnID uint64) uint64 { + return (rowID * SliceWidth) + (columnID % SliceWidth) } diff --git a/fragment_test.go b/fragment_test.go index f54f5415b..fd5dc95ac 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -35,19 +35,19 @@ func TestFragment_SetBit(t *testing.T) { t.Fatal(err) } - // Verify counts on bitmaps. - if n := f.Bitmap(120).Count(); n != 2 { + // Verify counts on rows. + if n := f.Row(120).Count(); n != 2 { t.Fatalf("unexpected count: %d", n) - } else if n := f.Bitmap(121).Count(); n != 1 { + } else if n := f.Row(121).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.Bitmap(120).Count(); n != 2 { + } else if n := f.Row(120).Count(); n != 2 { t.Fatalf("unexpected count (reopen): %d", n) - } else if n := f.Bitmap(121).Count(); n != 1 { + } else if n := f.Row(121).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -66,15 +66,15 @@ func TestFragment_ClearBit(t *testing.T) { t.Fatal(err) } - // Verify count on bitmap. - if n := f.Bitmap(1000).Count(); n != 1 { + // Verify count on row. + if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.Bitmap(1000).Count(); n != 1 { + } else if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -96,14 +96,14 @@ func TestFragment_Snapshot(t *testing.T) { // Snapshot bitmap and verify data. if err := f.Snapshot(); err != nil { t.Fatal(err) - } else if n := f.Bitmap(1000).Count(); n != 1 { + } else if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count: %d", n) } // Close and reopen the fragment & verify the data. if err := f.Reopen(); err != nil { t.Fatal(err) - } else if n := f.Bitmap(1000).Count(); n != 1 { + } else if n := f.Row(1000).Count(); n != 1 { t.Fatalf("unexpected count (reopen): %d", n) } } @@ -124,8 +124,8 @@ func TestFragment_ForEachBit(t *testing.T) { // Iterate over bits. var result [][2]uint64 - if err := f.ForEachBit(func(bitmapID, profileID uint64) error { - result = append(result, [2]uint64{bitmapID, profileID}) + if err := f.ForEachBit(func(rowID, columnID uint64) error { + result = append(result, [2]uint64{rowID, columnID}) return nil }); err != nil { t.Fatal(err) @@ -142,12 +142,12 @@ func TestFragment_Top(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Set bits on the bitmaps 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 200) f.MustSetBits(101, 1) f.MustSetBits(102, 1, 2) - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{N: 2}); err != nil { t.Fatal(err) } else if len(pairs) != 2 { @@ -159,21 +159,21 @@ func TestFragment_Top(t *testing.T) { } } -// Ensure a fragment can filter bitmaps when retrieving the top n bitmaps. +// Ensure a fragment can filter rows when retrieving the top n rows. func TestFragment_Top_Filter(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Set bits on the bitmaps 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 200) f.MustSetBits(101, 1) f.MustSetBits(102, 1, 2) // Assign attributes. - f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) - f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)}) + f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)}) + f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)}) - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{ N: 2, FilterField: "x", @@ -189,21 +189,21 @@ func TestFragment_Top_Filter(t *testing.T) { } } -// Ensure a fragment can return top bitmaps that intersect with an input bitmap. +// Ensure a fragment can return top rows that intersect with an input row. func TestFragment_TopN_Intersect(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Create an intersecting input bitmap. + // Create an intersecting input row. src := pilosa.NewBitmap(1, 2, 3) - // Set bits on various bitmaps. + // Set bits on various rows. f.MustSetBits(100, 1, 10, 11, 12) // one intersection f.MustSetBits(101, 1, 2, 3, 4) // three intersections f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections f.MustSetBits(103, 1000, 1001, 1002) // no intersection - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{N: 3, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ @@ -215,7 +215,7 @@ func TestFragment_TopN_Intersect(t *testing.T) { } } -// Ensure a fragment can return top bitmaps that have many bits set. +// Ensure a fragment can return top rows that have many bits set. func TestFragment_TopN_Intersect_Large(t *testing.T) { if testing.Short() { t.Skip("short mode") @@ -224,20 +224,20 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Create an intersecting input bitmap. + // Create an intersecting input row. src := pilosa.NewBitmap( 980, 981, 982, 983, 984, 985, 986, 987, 988, 989, 990, 991, 992, 993, 994, 995, 996, 997, 998, 999, ) - // Set bits on bitmaps 0 - 999. Higher bitmaps have higher bit counts. + // Set bits on rows 0 - 999. Higher rows have higher bit counts. for i := uint64(0); i < 1000; i++ { for j := uint64(0); j < i; j++ { f.MustSetBits(i, j) } } - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ @@ -256,18 +256,18 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) { } } -// Ensure a fragment can return top bitmaps when specified by ID. -func TestFragment_TopN_BitmapIDs(t *testing.T) { +// Ensure a fragment can return top rows when specified by ID. +func TestFragment_TopN_IDs(t *testing.T) { f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) defer f.Close() - // Set bits on various bitmaps. + // Set bits on various rows. f.MustSetBits(100, 1, 2, 3) f.MustSetBits(101, 4, 5, 6, 7) f.MustSetBits(102, 8, 9, 10, 11, 12) - // Retrieve top bitmaps. - if pairs, err := f.Top(pilosa.TopOptions{BitmapIDs: []uint64{100, 101, 200}}); err != nil { + // Retrieve top rows. + if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil { t.Fatal(err) } else if !reflect.DeepEqual(pairs, []pilosa.Pair{ {ID: 101, Count: 4}, @@ -307,16 +307,16 @@ func TestFragment_TopN_CacheSize(t *testing.T) { frag.Close() f := &Fragment{ - Fragment: frag, - BitmapAttrStore: MustOpenAttrStore(), + Fragment: frag, + RowAttrStore: MustOpenAttrStore(), } - f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore if err := f.Open(); err != nil { panic(err) } defer f.Close() - // Set bits on various bitmaps. + // Set bits on various rows. f.MustSetBits(100, 1, 2, 3) f.MustSetBits(101, 4, 5, 6, 7) f.MustSetBits(102, 8, 9, 10, 11, 12) @@ -332,7 +332,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) { {ID: 102, Count: 5}, } - // Retrieve top bitmaps. + // Retrieve top rows. if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil { t.Fatal(err) } else if len(pairs) > int(cacheSize) { @@ -381,7 +381,7 @@ func TestFragment_Blocks(t *testing.T) { } prev = blocks - // Set bit on different bitmap. + // Set bit on different row. if _, err := f.SetBit(20, 0); err != nil { t.Fatal(err) } @@ -391,7 +391,7 @@ func TestFragment_Blocks(t *testing.T) { } prev = blocks - // Set bit on different profile. + // Set bit on different column. if _, err := f.SetBit(20, 100); err != nil { t.Fatal(err) } @@ -544,7 +544,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify data in other fragment. - if a := f1.Bitmap(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { + if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected bits: %+v", a) } @@ -553,40 +553,11 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { t.Fatal(err) } else if n := f1.Cache().Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.Bitmap(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { + } else if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { t.Fatalf("unexpected bits (reopen): %+v", a) } } -/* -func BenchmarkFragment_BlockChecksum_Fill1(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.01) } -func BenchmarkFragment_BlockChecksum_Fill10(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.10) } -func BenchmarkFragment_BlockChecksum_Fill50(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.50) } - -func benchmarkFragmentBlockChecksum(b *testing.B, fillPercent float64) { - f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0) - defer f.Close() - - // Fill fragment. - bitmapIDs, profileIDs := GenerateImportFill(pilosa.HashBlockSize, fillPercent) - if err := f.Import(bitmapIDs, profileIDs); err != nil { - b.Fatal(err) - } - - b.ResetTimer() - b.ReportAllocs() - - // Calculate block checksum. - for i := 0; i < b.N; i++ { - f.InvalidateChecksums() - - if chksum := f.BlockChecksum(0); chksum == nil { - b.Fatal("expected checksum") - } - } -} -*/ - func BenchmarkFragment_Blocks(b *testing.B) { if *FragmentPath == "" { b.Skip("no fragment specified") @@ -633,7 +604,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Start benchmark b.ResetTimer() for i := 0; i < b.N; i++ { - if n := f.Bitmap(1).IntersectionCount(f.Bitmap(2)); n == 0 { + if n := f.Row(1).IntersectionCount(f.Row(2)); n == 0 { b.Fatalf("unexpected count: %d", n) } } @@ -642,7 +613,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) { // Fragment is a test wrapper for pilosa.Fragment. type Fragment struct { *pilosa.Fragment - BitmapAttrStore *AttrStore + RowAttrStore *AttrStore } // NewFragment returns a new instance of Fragment with a temporary path. @@ -654,10 +625,10 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment { file.Close() f := &Fragment{ - Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), - BitmapAttrStore: MustOpenAttrStore(), + Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice), + RowAttrStore: MustOpenAttrStore(), } - f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore return f } @@ -674,7 +645,7 @@ func MustOpenFragment(db, frame, view string, slice uint64) *Fragment { func (f *Fragment) Close() error { defer os.Remove(f.Path()) defer os.Remove(f.CachePath()) - defer f.BitmapAttrStore.Close() + defer f.RowAttrStore.Close() return f.Fragment.Close() } @@ -686,64 +657,64 @@ func (f *Fragment) Reopen() error { } f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice()) - f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore + f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore if err := f.Open(); err != nil { return err } return nil } -// MustSetBits sets bits on a bitmap. Panic on error. +// MustSetBits sets bits on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) { - for _, profileID := range profileIDs { - if _, err := f.SetBit(bitmapID, profileID); err != nil { +func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := f.SetBit(rowID, columnID); err != nil { panic(err) } } } -// MustClearBits clears bits on a bitmap. Panic on error. -func (f *Fragment) MustClearBits(bitmapID uint64, profileIDs ...uint64) { - for _, profileID := range profileIDs { - if _, err := f.ClearBit(bitmapID, profileID); err != nil { +// MustClearBits clears bits on a row. Panic on error. +func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := f.ClearBit(rowID, columnID); err != nil { panic(err) } } } -// BitmapAttrStore provides simple storage for attributes. -type BitmapAttrStore struct { +// RowAttrStore provides simple storage for attributes. +type RowAttrStore struct { attrs map[uint64]map[string]interface{} } -// NewBitmapAttrStore returns a new instance of BitmapAttrStore. -func NewBitmapAttrStore() *BitmapAttrStore { - return &BitmapAttrStore{ +// NewRowAttrStore returns a new instance of RowAttrStore. +func NewRowAttrStore() *RowAttrStore { + return &RowAttrStore{ attrs: make(map[uint64]map[string]interface{}), } } -// BitmapAttrs returns the attributes set to a bitmap id. -func (s *BitmapAttrStore) BitmapAttrs(id uint64) (map[string]interface{}, error) { +// RowAttrs returns the attributes set to a row id. +func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) { return s.attrs[id], nil } -// SetBitmapAttrs assigns a set of attributes to a bitmap id. -func (s *BitmapAttrStore) SetBitmapAttrs(id uint64, m map[string]interface{}) { +// SetRowAttrs assigns a set of attributes to a row id. +func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { s.attrs[id] = m } // GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk. -func GenerateImportFill(bitmapN int, pct float64) (bitmapIDs, profileIDs []uint64) { +func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { ipct := int(pct * 100) - for i := 0; i < SliceWidth*bitmapN; i++ { + for i := 0; i < SliceWidth*rowN; i++ { if i%100 >= ipct { continue } - bitmapIDs = append(bitmapIDs, uint64(i%SliceWidth)) - profileIDs = append(profileIDs, uint64(i/SliceWidth)) + rowIDs = append(rowIDs, uint64(i%SliceWidth)) + columnIDs = append(columnIDs, uint64(i/SliceWidth)) } return } @@ -754,7 +725,7 @@ func TestFragment_Tanimoto(t *testing.T) { src := pilosa.NewBitmap(1, 2, 3) - // Set bits on the bitmaps 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 2, 200) f.MustSetBits(101, 1, 3) f.MustSetBits(102, 1, 2, 10, 12) @@ -776,7 +747,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) { src := pilosa.NewBitmap(1, 2, 3) - // Set bits on the bitmaps 100, 101, & 102. + // Set bits on the rows 100, 101, & 102. f.MustSetBits(100, 1, 3, 2, 200) f.MustSetBits(101, 1, 3) f.MustSetBits(102, 1, 2, 10, 12) diff --git a/frame.go b/frame.go index 684b2225d..df4c8182f 100644 --- a/frame.go +++ b/frame.go @@ -35,8 +35,8 @@ type Frame struct { views map[string]*View - // Bitmap attribute storage and cache - bitmapAttrStore *AttrStore + // Row attribute storage and cache + rowAttrStore *AttrStore broadcaster Broadcaster stats StatsClient @@ -64,8 +64,8 @@ func NewFrame(path, db, name string) (*Frame, error) { db: db, name: name, - views: make(map[string]*View), - bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")), + views: make(map[string]*View), + rowAttrStore: NewAttrStore(filepath.Join(path, ".data")), stats: NopStatsClient, @@ -87,8 +87,8 @@ func (f *Frame) DB() string { return f.db } // Path returns the path the frame was initialized with. func (f *Frame) Path() string { return f.path } -// BitmapAttrStore returns the attribute storage. -func (f *Frame) BitmapAttrStore() *AttrStore { return f.bitmapAttrStore } +// RowAttrStore returns the attribute storage. +func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore } // MaxSlice returns the max slice in the frame. func (f *Frame) MaxSlice() uint64 { @@ -215,7 +215,7 @@ func (f *Frame) Open() error { return err } - if err := f.bitmapAttrStore.Open(); err != nil { + if err := f.rowAttrStore.Open(); err != nil { return err } @@ -253,7 +253,7 @@ func (f *Frame) openViews() error { if err := view.Open(); err != nil { return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err) } - view.BitmapAttrStore = f.bitmapAttrStore + view.RowAttrStore = f.rowAttrStore f.views[view.Name()] = view f.stats.Count("maxSlice", 1) @@ -326,8 +326,8 @@ func (f *Frame) Close() error { defer f.mu.Unlock() // Close the attribute store. - if f.bitmapAttrStore != nil { - _ = f.bitmapAttrStore.Close() + if f.rowAttrStore != nil { + _ = f.rowAttrStore.Close() } // Close all views. @@ -411,7 +411,7 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { if err := view.Open(); err != nil { return nil, err } - view.BitmapAttrStore = f.bitmapAttrStore + view.RowAttrStore = f.rowAttrStore f.views[view.Name()] = view return view, nil @@ -421,7 +421,7 @@ func (f *Frame) newView(path, name string) *View { view := NewView(path, f.db, f.name, name, f.cacheSize) view.cacheType = f.cacheType view.LogOutput = f.LogOutput - view.BitmapAttrStore = f.bitmapAttrStore + view.RowAttrStore = f.rowAttrStore view.stats = f.stats.WithTags(fmt.Sprintf("slice:%s", name)) return view } @@ -511,7 +511,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change } // Import bulk imports data. -func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error { +func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error { // Determine quantum if timestamps are set. q := f.TimeQuantum() if hasTime(timestamps) && q == "" { @@ -520,8 +520,8 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) // Split import data by fragment. dataByFragment := make(map[importKey]importData) - for i := range bitmapIDs { - bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i] + for i := range rowIDs { + rowID, columnID, timestamp := rowIDs[i], columnIDs[i], timestamps[i] var standard, inverse []string if timestamp == nil { @@ -534,20 +534,20 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) // Attach bit to each standard view. for _, name := range standard { - key := importKey{View: name, Slice: profileID / SliceWidth} + key := importKey{View: name, Slice: columnID / SliceWidth} data := dataByFragment[key] - data.BitmapIDs = append(data.BitmapIDs, bitmapID) - data.ProfileIDs = append(data.ProfileIDs, profileID) + data.RowIDs = append(data.RowIDs, rowID) + data.ColumnIDs = append(data.ColumnIDs, columnID) dataByFragment[key] = data } if f.inverseEnabled { // Attach reversed bits to each inverse view. for _, name := range inverse { - key := importKey{View: name, Slice: bitmapID / SliceWidth} + key := importKey{View: name, Slice: rowID / SliceWidth} data := dataByFragment[key] - data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed - data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed + data.RowIDs = append(data.RowIDs, columnID) // reversed + data.ColumnIDs = append(data.ColumnIDs, rowID) // reversed dataByFragment[key] = data } } @@ -563,8 +563,8 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) // Re-sort data for inverse views. if IsInverseView(key.View) { sort.Sort(importBitSet{ - bitmapIDs: data.BitmapIDs, - profileIDs: data.ProfileIDs, + rowIDs: data.RowIDs, + columnIDs: data.ColumnIDs, }) } @@ -578,7 +578,7 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) return err } - if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil { + if err := frag.Import(data.RowIDs, data.ColumnIDs); err != nil { return err } } @@ -650,15 +650,15 @@ func (o *FrameOptions) Encode() *internal.FrameMeta { // importBitSet represents slices of row and column ids. // This is used to sort data during import. type importBitSet struct { - bitmapIDs, profileIDs []uint64 + rowIDs, columnIDs []uint64 } func (p importBitSet) Swap(i, j int) { - p.bitmapIDs[i], p.bitmapIDs[j] = p.bitmapIDs[j], p.bitmapIDs[i] - p.profileIDs[i], p.profileIDs[j] = p.profileIDs[j], p.profileIDs[i] + p.rowIDs[i], p.rowIDs[j] = p.rowIDs[j], p.rowIDs[i] + p.columnIDs[i], p.columnIDs[j] = p.columnIDs[j], p.columnIDs[i] } -func (p importBitSet) Len() int { return len(p.bitmapIDs) } -func (p importBitSet) Less(i, j int) bool { return p.bitmapIDs[i] < p.bitmapIDs[j] } +func (p importBitSet) Len() int { return len(p.rowIDs) } +func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] } // Cache types. const ( diff --git a/frame_test.go b/frame_test.go index 4814ca67e..0d10841db 100644 --- a/frame_test.go +++ b/frame_test.go @@ -119,8 +119,8 @@ func (f *Frame) Reopen() error { } // MustSetBit sets a bit on the frame. Panic on error. -func (f *Frame) MustSetBit(view string, bitmapID, profileID uint64, t *time.Time) (changed bool) { - changed, err := f.SetBit(view, bitmapID, profileID, t) +func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) { + changed, err := f.SetBit(view, rowID, columnID, t) if err != nil { panic(err) } diff --git a/handler.go b/handler.go index ef5ad4f8d..7040a05ab 100644 --- a/handler.go +++ b/handler.go @@ -159,26 +159,26 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { results, err := h.Executor.Execute(r.Context(), dbName, q, req.Slices, opt) resp := &QueryResponse{Results: results, Err: err} - // Fill profile attributes if requested. - if req.Profiles { - // Consolidate all profile ids across all calls. - var profileIDs []uint64 + // Fill column attributes if requested. + if req.ColumnAttrs { + // Consolidate all column ids across all calls. + var columnIDs []uint64 for _, result := range results { bm, ok := result.(*Bitmap) if !ok { continue } - profileIDs = uint64Slice(profileIDs).merge(bm.Bits()) + columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) } - // Retrieve profile attributes across all calls. - profiles, err := h.readProfiles(h.Index.DB(dbName), profileIDs) + // Retrieve column attributes across all calls. + columnAttrSets, err := h.readColumnAttrSets(h.Index.DB(dbName), columnIDs) if err != nil { w.WriteHeader(http.StatusInternalServerError) h.writeQueryResponse(w, r, &QueryResponse{Err: err}) return } - resp.Profiles = profiles + resp.ColumnAttrSets = columnAttrSets } // Set appropriate status code, if there is an error. @@ -437,7 +437,7 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { } // Retrieve local blocks. - blks, err := db.ProfileAttrStore().Blocks() + blks, err := db.ColumnAttrStore().Blocks() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -447,7 +447,7 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) { attrs := make(map[uint64]map[string]interface{}) for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { // Retrieve block data. - m, err := db.ProfileAttrStore().BlockData(blockID) + m, err := db.ColumnAttrStore().BlockData(blockID) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -701,7 +701,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request } // Retrieve local blocks. - blks, err := f.BitmapAttrStore().Blocks() + blks, err := f.RowAttrStore().Blocks() if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -711,7 +711,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request attrs := make(map[uint64]map[string]interface{}) for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) { // Retrieve block data. - m, err := f.BitmapAttrStore().BlockData(blockID) + m, err := f.RowAttrStore().BlockData(blockID) if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return @@ -739,24 +739,24 @@ type postFrameAttrDiffResponse struct { Attrs map[uint64]map[string]interface{} `json:"attrs"` } -// readProfiles returns a list of profile objects by id. -func (h *Handler) readProfiles(db *DB, ids []uint64) ([]*Profile, error) { +// readColumnAttrSets returns a list of column attribute objects by id. +func (h *Handler) readColumnAttrSets(db *DB, ids []uint64) ([]*ColumnAttrSet, error) { if db == nil { return nil, nil } - a := make([]*Profile, 0, len(ids)) + a := make([]*ColumnAttrSet, 0, len(ids)) for _, id := range ids { - // Read attributes for profile. Skip profile if empty. - attrs, err := db.ProfileAttrStore().Attrs(id) + // Read attributes for column. Skip column if empty. + attrs, err := db.ColumnAttrStore().Attrs(id) if err != nil { return nil, err } else if len(attrs) == 0 { continue } - // Append profile with attributes. - a = append(a, &Profile{ID: id, Attrs: attrs}) + // Append column with attributes. + a = append(a, &ColumnAttrSet{ID: id, Attrs: attrs}) } return a, nil @@ -817,10 +817,10 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { } return &QueryRequest{ - Query: query, - Slices: slices, - Profiles: q.Get("profiles") == "true", - Quantum: quantum, + Query: query, + Slices: slices, + ColumnAttrs: q.Get("columnAttrs") == "true", + Quantum: quantum, }, nil } @@ -907,9 +907,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) { } // Import into fragment. - err = f.Import(req.BitmapIDs, req.ProfileIDs, timestamps) + err = f.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err) + h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ColumnIDs), err) return } @@ -965,10 +965,10 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) { cw := csv.NewWriter(w) // Iterate over each bit. - if err := f.ForEachBit(func(bitmapID, profileID uint64) error { + if err := f.ForEachBit(func(rowID, columnID uint64) error { return cw.Write([]string{ - strconv.FormatUint(bitmapID, 10), - strconv.FormatUint(profileID, 10), + strconv.FormatUint(rowID, 10), + strconv.FormatUint(columnID, 10), }) }); err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) @@ -1083,7 +1083,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ // Read data var resp internal.BlockDataResponse if f != nil { - resp.BitmapIDs, resp.ProfileIDs = f.BlockData(int(req.Block)) + resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block)) } // Encode response. @@ -1271,8 +1271,8 @@ type QueryRequest struct { // If empty, all slices are included. Slices []uint64 - // Return profile attributes, if true. - Profiles bool + // Return column attributes, if true. + ColumnAttrs bool // Time granularity to use with the timestamp. Quantum TimeQuantum @@ -1284,11 +1284,11 @@ type QueryRequest struct { func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { req := &QueryRequest{ - Query: pb.Query, - Slices: pb.Slices, - Profiles: pb.Profiles, - Quantum: TimeQuantum(pb.Quantum), - Remote: pb.Remote, + Query: pb.Query, + Slices: pb.Slices, + ColumnAttrs: pb.ColumnAttrs, + Quantum: TimeQuantum(pb.Quantum), + Remote: pb.Remote, } return req @@ -1300,8 +1300,8 @@ type QueryResponse struct { // Can be a Bitmap, Pairs, or uint64. Results []interface{} - // Set of profiles matching IDs returned in Result. - Profiles []*Profile + // Set of column attribute objects matching IDs returned in Result. + ColumnAttrSets []*ColumnAttrSet // Error during parsing or execution. Err error @@ -1309,12 +1309,12 @@ type QueryResponse struct { func (resp *QueryResponse) MarshalJSON() ([]byte, error) { var output struct { - Results []interface{} `json:"results,omitempty"` - Profiles []*Profile `json:"profiles,omitempty"` - Err string `json:"error,omitempty"` + Results []interface{} `json:"results,omitempty"` + ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"` + Err string `json:"error,omitempty"` } output.Results = resp.Results - output.Profiles = resp.Profiles + output.ColumnAttrSets = resp.ColumnAttrSets if resp.Err != nil { output.Err = resp.Err.Error() @@ -1324,8 +1324,8 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) { func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse { pb := &internal.QueryResponse{ - Results: make([]*internal.QueryResult, len(resp.Results)), - Profiles: encodeProfiles(resp.Profiles), + Results: make([]*internal.QueryResult, len(resp.Results)), + ColumnAttrSets: encodeColumnAttrSets(resp.ColumnAttrSets), } for i := range resp.Results { diff --git a/handler_test.go b/handler_test.go index 049fdc036..af4da1dd7 100644 --- a/handler_test.go +++ b/handler_test.go @@ -253,18 +253,18 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { } } -// Ensure the handler can execute a query that returns a bitmap with profiles as JSON. -func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { +// Ensure the handler can execute a query that returns a bitmap with column attributes as JSON. +func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) { idx := NewIndex() defer idx.Close() - // Create database and set profile attributes. + // Create database and set column attributes. db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil { t.Fatal(err) } @@ -277,10 +277,10 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) { } w := httptest.NewRecorder() - h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query?profiles=true", strings.NewReader("Bitmap(id=100)"))) + h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -318,16 +318,16 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) { } } -// Ensure the handler can execute a query that returns a bitmap with profiles as protobuf. -func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { +// Ensure the handler can execute a query that returns a bitmap with column attributes as protobuf. +func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) { idx := NewIndex() defer idx.Close() - // Create database and set profile attributes. + // Create database and set column attributes. db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{}) if err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) } @@ -341,8 +341,8 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { // Encode request body. buf, err := proto.Marshal(&internal.QueryRequest{ - Query: "Bitmap(id=100)", - Profiles: true, + Query: "Bitmap(id=100)", + ColumnAttrs: true, }) if err != nil { t.Fatal(err) @@ -373,12 +373,12 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) { t.Fatalf("unexpected attr[2]: %s=%v", k, v) } - if a := resp.Profiles; len(a) != 1 { - t.Fatalf("unexpected profiles length: %d", len(a)) + if a := resp.ColumnAttrSets; len(a) != 1 { + t.Fatalf("unexpected column attributes length: %d", len(a)) } else if a[0].ID != 1 { t.Fatalf("unexpected id: %d", a[0].ID) } else if len(a[0].Attrs) != 1 { - t.Fatalf("unexpected profile attr length: %d", len(a)) + t.Fatalf("unexpected column attr length: %d", len(a)) } else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" { t.Fatalf("unexpected attr[0]: %s=%v", k, v) } @@ -603,16 +603,16 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) { if err != nil { t.Fatal(err) } - if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := db.ProfileAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + } else if err := db.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { t.Fatal(err) } // Retrieve block checksums. - blks, err := db.ProfileAttrStore().Blocks() + blks, err := db.ColumnAttrStore().Blocks() if err != nil { t.Fatal(err) } @@ -653,16 +653,16 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) { if err != nil { t.Fatal(err) } - if err := f.BitmapAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { + if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil { t.Fatal(err) - } else if err := f.BitmapAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { + } else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil { t.Fatal(err) - } else if err := f.BitmapAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { + } else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil { t.Fatal(err) } // Retrieve block checksums. - blks, err := f.BitmapAttrStore().Blocks() + blks, err := f.RowAttrStore().Blocks() if err != nil { t.Fatal(err) } @@ -732,7 +732,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) { f1 := idx.Fragment("x", "y", pilosa.ViewStandard, 0) if f1 == nil { t.Fatal("fragment x/y/standard/0 not created") - } else if bits := f1.Bitmap(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) { + } else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) { t.Fatalf("unexpected restored bits: %+v", bits) } } diff --git a/index.go b/index.go index eb8f4b204..93228bb14 100644 --- a/index.go +++ b/index.go @@ -34,7 +34,7 @@ type Index struct { // Data directory path. Path string - // The interval at which the cached bitmap ids are persisted to disk. + // The interval at which the cached row ids are persisted to disk. CacheFlushInterval time.Duration LogOutput io.Writer @@ -375,7 +375,7 @@ func (s *IndexSyncer) SyncIndex() error { return nil } - // Sync database profile attributes. + // Sync database column attributes. if err := s.syncDatabase(di.Name); err != nil { return fmt.Errorf("db sync error: db=%s, err=%s", di.Name, err) } @@ -386,7 +386,7 @@ func (s *IndexSyncer) SyncIndex() error { return nil } - // Sync frame bitmap attributes. + // Sync frame row attributes. if err := s.syncFrame(di.Name, fi.Name); err != nil { return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err) } @@ -429,7 +429,7 @@ func (s *IndexSyncer) syncDatabase(db string) error { } // Read block checksums. - blks, err := d.ProfileAttrStore().Blocks() + blks, err := d.ColumnAttrStore().Blocks() if err != nil { return err } @@ -443,7 +443,7 @@ func (s *IndexSyncer) syncDatabase(db string) error { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.ProfileAttrDiff(context.Background(), db, blks) + m, err := client.ColumnAttrDiff(context.Background(), db, blks) if err != nil { return err } else if len(m) == 0 { @@ -451,12 +451,12 @@ func (s *IndexSyncer) syncDatabase(db string) error { } // Update local copy. - if err := d.ProfileAttrStore().SetBulkAttrs(m); err != nil { + if err := d.ColumnAttrStore().SetBulkAttrs(m); err != nil { return err } // Recompute blocks. - blks, err = d.ProfileAttrStore().Blocks() + blks, err = d.ColumnAttrStore().Blocks() if err != nil { return err } @@ -474,7 +474,7 @@ func (s *IndexSyncer) syncFrame(db, name string) error { } // Read block checksums. - blks, err := f.BitmapAttrStore().Blocks() + blks, err := f.RowAttrStore().Blocks() if err != nil { return err } @@ -488,7 +488,7 @@ func (s *IndexSyncer) syncFrame(db, name string) error { // Retrieve attributes from differing blocks. // Skip update and recomputation if no attributes have changed. - m, err := client.BitmapAttrDiff(context.Background(), db, name, blks) + m, err := client.RowAttrDiff(context.Background(), db, name, blks) if err == ErrFrameNotFound { continue // frame not created remotely yet, skip } else if err != nil { @@ -498,12 +498,12 @@ func (s *IndexSyncer) syncFrame(db, name string) error { } // Update local copy. - if err := f.BitmapAttrStore().SetBulkAttrs(m); err != nil { + if err := f.RowAttrStore().SetBulkAttrs(m); err != nil { return err } // Recompute blocks. - blks, err = f.BitmapAttrStore().Blocks() + blks, err = f.RowAttrStore().Blocks() if err != nil { return err } diff --git a/index_test.go b/index_test.go index 4a1be4cf2..22efbc718 100644 --- a/index_test.go +++ b/index_test.go @@ -135,28 +135,28 @@ func TestIndexSyncer_SyncIndex(t *testing.T) { // Verify data is the same on both nodes. for i, idx := range []*Index{idx0, idx1} { f := idx.Fragment("d", "f", pilosa.ViewStandard, 0) - if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { t.Fatalf("unexpected bits(%d/0): %+v", i, a) - } else if a := f.Bitmap(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { + } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { t.Fatalf("unexpected bits(%d/2): %+v", i, a) - } else if a := f.Bitmap(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected bits(%d/3): %+v", i, a) - } else if a := f.Bitmap(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { + } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { t.Fatalf("unexpected bits(%d/120): %+v", i, a) - } else if a := f.Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { + } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { t.Fatalf("unexpected bits(%d/200): %+v", i, a) } f = idx.Fragment("d", "f0", pilosa.ViewStandard, 1) - a := f.Bitmap(9).Bits() + a := f.Row(9).Bits() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) } - if a := f.Bitmap(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) } f = idx.Fragment("y", "z", pilosa.ViewStandard, 3) - if a := f.Bitmap(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) } } diff --git a/internal/private.pb.go b/internal/private.pb.go index 0423cfa08..f82ba23f8 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -90,8 +90,8 @@ func (*BlockDataRequest) ProtoMessage() {} func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{3} } type BlockDataResponse struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` + RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` } func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} } @@ -100,7 +100,7 @@ func (*BlockDataResponse) ProtoMessage() {} func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{4} } type Cache struct { - BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` + IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"` } func (m *Cache) Reset() { *m = Cache{} } @@ -432,10 +432,10 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.BitmapIDs) > 0 { - dAtA2 := make([]byte, len(m.BitmapIDs)*10) + if len(m.RowIDs) > 0 { + dAtA2 := make([]byte, len(m.RowIDs)*10) var j1 int - for _, num := range m.BitmapIDs { + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -449,10 +449,10 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(j1)) i += copy(dAtA[i:], dAtA2[:j1]) } - if len(m.ProfileIDs) > 0 { - dAtA4 := make([]byte, len(m.ProfileIDs)*10) + if len(m.ColumnIDs) > 0 { + dAtA4 := make([]byte, len(m.ColumnIDs)*10) var j3 int - for _, num := range m.ProfileIDs { + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -484,10 +484,10 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.BitmapIDs) > 0 { - dAtA6 := make([]byte, len(m.BitmapIDs)*10) + if len(m.IDs) > 0 { + dAtA6 := make([]byte, len(m.IDs)*10) var j5 int - for _, num := range m.BitmapIDs { + for _, num := range m.IDs { for num >= 1<<7 { dAtA6[j5] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -924,16 +924,16 @@ func (m *BlockDataRequest) Size() (n int) { func (m *BlockDataResponse) Size() (n int) { var l int _ = l - if len(m.BitmapIDs) > 0 { + if len(m.RowIDs) > 0 { l = 0 - for _, e := range m.BitmapIDs { + for _, e := range m.RowIDs { l += sovPrivate(uint64(e)) } n += 1 + sovPrivate(uint64(l)) + l } - if len(m.ProfileIDs) > 0 { + if len(m.ColumnIDs) > 0 { l = 0 - for _, e := range m.ProfileIDs { + for _, e := range m.ColumnIDs { l += sovPrivate(uint64(e)) } n += 1 + sovPrivate(uint64(l)) + l @@ -944,9 +944,9 @@ func (m *BlockDataResponse) Size() (n int) { func (m *Cache) Size() (n int) { var l int _ = l - if len(m.BitmapIDs) > 0 { + if len(m.IDs) > 0 { l = 0 - for _, e := range m.BitmapIDs { + for _, e := range m.IDs { l += sovPrivate(uint64(e)) } n += 1 + sovPrivate(uint64(l)) + l @@ -1714,7 +1714,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.RowIDs = append(m.RowIDs, v) } } else if wireType == 0 { var v uint64 @@ -1732,9 +1732,9 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.RowIDs = append(m.RowIDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) } case 2: if wireType == 2 { @@ -1776,7 +1776,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.ProfileIDs = append(m.ProfileIDs, v) + m.ColumnIDs = append(m.ColumnIDs, v) } } else if wireType == 0 { var v uint64 @@ -1794,9 +1794,9 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { break } } - m.ProfileIDs = append(m.ProfileIDs, v) + m.ColumnIDs = append(m.ColumnIDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } default: iNdEx = preIndex @@ -1888,7 +1888,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.IDs = append(m.IDs, v) } } else if wireType == 0 { var v uint64 @@ -1906,9 +1906,9 @@ func (m *Cache) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.IDs = append(m.IDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field IDs", wireType) } default: iNdEx = preIndex @@ -3146,43 +3146,43 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 600 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcd, 0x4e, 0x14, 0x41, - 0x10, 0x76, 0x7e, 0x20, 0x4c, 0x21, 0xcb, 0xd2, 0x7a, 0x98, 0x10, 0x32, 0x59, 0x3b, 0x2a, 0xc4, - 0x03, 0x07, 0xbc, 0x18, 0xe2, 0x69, 0x18, 0x14, 0x12, 0x20, 0xd2, 0x8b, 0xde, 0x7b, 0x97, 0x52, - 0x27, 0x3b, 0x7f, 0xce, 0xf4, 0x2e, 0xac, 0x57, 0x5f, 0xc2, 0xc4, 0x67, 0xf0, 0x3d, 0x3c, 0xfa, - 0x08, 0x66, 0x7d, 0x11, 0xd3, 0xdd, 0xf3, 0xe7, 0xb2, 0xf8, 0x73, 0xeb, 0xfa, 0xaa, 0xea, 0xab, - 0xaf, 0xbf, 0xa9, 0x1e, 0x58, 0xcb, 0xf2, 0x70, 0xc2, 0x05, 0xee, 0x66, 0x79, 0x2a, 0x52, 0xb2, - 0x12, 0x26, 0x02, 0xf3, 0x84, 0x47, 0xf4, 0x04, 0x96, 0x03, 0xff, 0x14, 0x05, 0x27, 0x3d, 0x58, - 0x3d, 0x48, 0xa3, 0x71, 0x9c, 0x9c, 0xf0, 0x01, 0x46, 0xae, 0xd1, 0x33, 0x76, 0x1c, 0xd6, 0x86, - 0x64, 0xc5, 0x45, 0x18, 0xe3, 0xf9, 0x98, 0x27, 0x62, 0x1c, 0xbb, 0xa6, 0xae, 0x68, 0x41, 0xf4, - 0xab, 0x01, 0xce, 0x8b, 0x9c, 0xc7, 0xa8, 0x18, 0x37, 0x61, 0x85, 0xa5, 0x57, 0x6d, 0xba, 0x3a, - 0x26, 0x8f, 0xa1, 0x73, 0x9c, 0x4c, 0x30, 0x2f, 0xf0, 0x30, 0xe1, 0x83, 0x08, 0x2f, 0x15, 0xdd, - 0x0a, 0x9b, 0x43, 0xc9, 0x16, 0x38, 0x07, 0x7c, 0xf8, 0x1e, 0x2f, 0xa6, 0x19, 0xba, 0x96, 0x22, - 0x69, 0x80, 0x3a, 0xdb, 0x0f, 0x3f, 0xa2, 0x6b, 0xf7, 0x8c, 0x9d, 0x35, 0xd6, 0x00, 0xf3, 0x7a, - 0x97, 0x6e, 0xea, 0xa5, 0xd0, 0x39, 0x8e, 0xb3, 0x34, 0x17, 0x0c, 0x8b, 0x2c, 0x4d, 0x0a, 0x24, - 0x5d, 0xb0, 0x0e, 0xf3, 0xbc, 0x94, 0x2b, 0x8f, 0xf4, 0x1a, 0xba, 0x7e, 0x94, 0x0e, 0x47, 0x01, - 0x17, 0x9c, 0xe1, 0x87, 0x31, 0x16, 0x82, 0x74, 0xc0, 0x0c, 0xfc, 0xb2, 0xc8, 0x0c, 0x7c, 0x72, - 0x1f, 0x96, 0xd4, 0xb5, 0x4b, 0x4f, 0x74, 0x20, 0x51, 0xd5, 0xa9, 0x74, 0xdb, 0x4c, 0x07, 0x12, - 0xed, 0x47, 0xe1, 0x50, 0xeb, 0xb5, 0x99, 0x0e, 0x08, 0x01, 0xfb, 0x4d, 0x88, 0x57, 0xa5, 0x48, - 0x75, 0xa6, 0xe7, 0xb0, 0xd1, 0x9a, 0x5c, 0x0a, 0xdc, 0x02, 0xc7, 0x0f, 0x45, 0xcc, 0xb3, 0xe3, - 0xa0, 0x70, 0x8d, 0x9e, 0xb5, 0x63, 0xb3, 0x06, 0x20, 0x1e, 0xc0, 0xab, 0x3c, 0x7d, 0x1b, 0x46, - 0x28, 0xd3, 0xa6, 0x4a, 0xb7, 0x10, 0xfa, 0x08, 0x96, 0x94, 0x3f, 0x7f, 0xa6, 0xa1, 0x5f, 0x0c, - 0xd8, 0x38, 0xe5, 0xd7, 0x4a, 0x5a, 0x51, 0x8f, 0x3e, 0x02, 0xa7, 0x06, 0x55, 0xcf, 0xea, 0xde, - 0x93, 0xdd, 0x6a, 0x93, 0x76, 0x6f, 0xd4, 0x37, 0xc8, 0x61, 0x22, 0xf2, 0x29, 0x6b, 0x9a, 0x37, - 0x9f, 0x43, 0xe7, 0xf7, 0xa4, 0xf4, 0x7d, 0x84, 0xd3, 0xca, 0xf7, 0x11, 0x4e, 0xa5, 0x4f, 0x13, - 0x1e, 0x8d, 0xb5, 0xa7, 0x36, 0xd3, 0xc1, 0xbe, 0xf9, 0xcc, 0xa0, 0xfb, 0x40, 0x0e, 0x72, 0xe4, - 0x02, 0x15, 0xc1, 0x29, 0x16, 0x05, 0x7f, 0x87, 0x8b, 0xbe, 0x89, 0xf6, 0xd9, 0x6c, 0xf9, 0x4c, - 0x1f, 0xc0, 0x7a, 0x80, 0x11, 0x0a, 0x94, 0x5b, 0xbf, 0xb0, 0x91, 0xbe, 0x84, 0x75, 0x4d, 0x7f, - 0x6b, 0x09, 0x79, 0x08, 0xb6, 0xdc, 0x70, 0x45, 0xbd, 0xba, 0xd7, 0x6d, 0x4c, 0xd0, 0x6f, 0x89, - 0xa9, 0x2c, 0x1d, 0x56, 0x3a, 0xcb, 0x27, 0x71, 0xab, 0xce, 0x05, 0xbb, 0xb3, 0x5d, 0x4e, 0xb0, - 0xd4, 0x84, 0x7b, 0xcd, 0x84, 0xfa, 0x79, 0x95, 0x43, 0xf6, 0x81, 0xe8, 0x0b, 0xfd, 0xff, 0x10, - 0x1a, 0x94, 0xa8, 0xdc, 0xbe, 0x33, 0x99, 0xd5, 0x0d, 0xea, 0x5c, 0x2b, 0x30, 0xff, 0xa6, 0xe0, - 0x93, 0x21, 0x87, 0x2d, 0xe4, 0xf8, 0x27, 0x9f, 0xe4, 0x7f, 0xa2, 0xda, 0x86, 0xf2, 0xa9, 0xd4, - 0x31, 0xd9, 0x86, 0x65, 0x35, 0xaf, 0x70, 0x6d, 0xb5, 0x70, 0xeb, 0x73, 0x3a, 0x58, 0x99, 0xa6, - 0xaf, 0xc1, 0x39, 0x4b, 0x2f, 0xb1, 0x2f, 0xb8, 0x50, 0xf7, 0x39, 0x4a, 0x0b, 0x51, 0x69, 0x91, - 0x67, 0xb5, 0x0f, 0x32, 0x59, 0x59, 0xa0, 0x2b, 0x3d, 0xb0, 0x02, 0xbf, 0x70, 0x2d, 0x45, 0x7e, - 0xb7, 0x2d, 0x90, 0xc9, 0x84, 0xdf, 0xfd, 0x36, 0xf3, 0x8c, 0xef, 0x33, 0xcf, 0xf8, 0x31, 0xf3, - 0x8c, 0xcf, 0x3f, 0xbd, 0x3b, 0x83, 0x65, 0xf5, 0x0b, 0x7d, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff, - 0x3a, 0x23, 0x0f, 0xb4, 0x53, 0x05, 0x00, 0x00, + // 596 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0x13, 0x41, + 0x14, 0x76, 0x7f, 0x68, 0xe8, 0x41, 0x4a, 0x19, 0x8d, 0x59, 0x89, 0x69, 0xea, 0xc4, 0x08, 0xf1, + 0x82, 0x0b, 0xbc, 0x31, 0xc4, 0xab, 0x65, 0x51, 0x9a, 0x00, 0x89, 0x03, 0x7a, 0x3f, 0x94, 0x13, + 0xdd, 0xb0, 0xdd, 0xad, 0xbb, 0x53, 0xa0, 0xde, 0xfa, 0x12, 0x26, 0x3e, 0x83, 0xef, 0xe1, 0xa5, + 0x8f, 0x60, 0xea, 0x8b, 0x98, 0x39, 0x33, 0xfb, 0x63, 0x29, 0x51, 0xef, 0xe6, 0x7c, 0xe7, 0xef, + 0x9b, 0x6f, 0xbf, 0x59, 0x58, 0x1d, 0xe7, 0xf1, 0xa5, 0x54, 0xb8, 0x3d, 0xce, 0x33, 0x95, 0xb1, + 0xe5, 0x38, 0x55, 0x98, 0xa7, 0x32, 0xe1, 0x87, 0xd0, 0x8a, 0xc2, 0x23, 0x54, 0x92, 0xf5, 0x61, + 0x65, 0x2f, 0x4b, 0x26, 0xa3, 0xf4, 0x50, 0x9e, 0x61, 0x12, 0x38, 0x7d, 0x67, 0xab, 0x2d, 0x9a, + 0x90, 0xae, 0x38, 0x8d, 0x47, 0xf8, 0x66, 0x22, 0x53, 0x35, 0x19, 0x05, 0xae, 0xa9, 0x68, 0x40, + 0xfc, 0x9b, 0x03, 0xed, 0x57, 0xb9, 0x1c, 0x21, 0x4d, 0xdc, 0x80, 0x65, 0x91, 0x5d, 0x35, 0xc7, + 0x55, 0x31, 0x7b, 0x0a, 0x9d, 0x41, 0x7a, 0x89, 0x79, 0x81, 0xfb, 0xa9, 0x3c, 0x4b, 0xf0, 0x9c, + 0xc6, 0x2d, 0x8b, 0x39, 0x94, 0x3d, 0x82, 0xf6, 0x9e, 0x1c, 0x7e, 0xc0, 0xd3, 0xe9, 0x18, 0x03, + 0x8f, 0x86, 0xd4, 0x40, 0x95, 0x3d, 0x89, 0x3f, 0x61, 0xe0, 0xf7, 0x9d, 0xad, 0x55, 0x51, 0x03, + 0xf3, 0x7c, 0x97, 0x6e, 0xf2, 0xe5, 0xd0, 0x19, 0x8c, 0xc6, 0x59, 0xae, 0x04, 0x16, 0xe3, 0x2c, + 0x2d, 0x90, 0x75, 0xc1, 0xdb, 0xcf, 0x73, 0x4b, 0x57, 0x1f, 0xf9, 0x35, 0x74, 0xc3, 0x24, 0x1b, + 0x5e, 0x44, 0x52, 0x49, 0x81, 0x1f, 0x27, 0x58, 0x28, 0xd6, 0x01, 0x37, 0x0a, 0x6d, 0x91, 0x1b, + 0x85, 0xec, 0x3e, 0x2c, 0xd1, 0xb5, 0xad, 0x26, 0x26, 0xd0, 0x28, 0x75, 0x12, 0x6f, 0x5f, 0x98, + 0x40, 0xa3, 0x27, 0x49, 0x3c, 0x34, 0x7c, 0x7d, 0x61, 0x02, 0xc6, 0xc0, 0x7f, 0x17, 0xe3, 0x95, + 0x25, 0x49, 0x67, 0x3e, 0x80, 0xf5, 0xc6, 0x66, 0x4b, 0xf0, 0x01, 0xb4, 0x44, 0x76, 0x35, 0x88, + 0x8a, 0xc0, 0xe9, 0x7b, 0x5b, 0xbe, 0xb0, 0x11, 0x49, 0x41, 0xdf, 0x4a, 0xa7, 0x5c, 0x4a, 0xd5, + 0x00, 0x7f, 0x08, 0x4b, 0xa4, 0x8b, 0xbe, 0x5f, 0xdd, 0xab, 0x8f, 0xfc, 0xab, 0x03, 0xeb, 0x47, + 0xf2, 0x9a, 0x68, 0x14, 0xd5, 0x9a, 0x03, 0x68, 0x57, 0x20, 0x55, 0xaf, 0xec, 0x3c, 0xdb, 0x2e, + 0x5d, 0xb3, 0x7d, 0xa3, 0xbe, 0x46, 0xf6, 0x53, 0x95, 0x4f, 0x45, 0xdd, 0xbc, 0xf1, 0x12, 0x3a, + 0x7f, 0x26, 0x35, 0x87, 0x0b, 0x9c, 0x96, 0x1a, 0x5f, 0xe0, 0x54, 0x6b, 0x72, 0x29, 0x93, 0x89, + 0xd1, 0xcf, 0x17, 0x26, 0xd8, 0x75, 0x5f, 0x38, 0x7c, 0x17, 0xd8, 0x5e, 0x8e, 0x52, 0x21, 0x0d, + 0x38, 0xc2, 0xa2, 0x90, 0xef, 0x71, 0x91, 0xfe, 0x46, 0x53, 0xb7, 0xa1, 0x29, 0x7f, 0x0c, 0x6b, + 0x11, 0x26, 0xa8, 0x50, 0x3b, 0x7c, 0x61, 0x23, 0x7f, 0x0d, 0x6b, 0x66, 0xfc, 0xad, 0x25, 0xec, + 0x09, 0xf8, 0xda, 0xcd, 0x34, 0x7a, 0x65, 0xa7, 0x5b, 0x8b, 0x60, 0xde, 0x8d, 0xa0, 0x2c, 0x1f, + 0x96, 0x3c, 0xad, 0xfd, 0x6f, 0xe5, 0xb9, 0xc0, 0x27, 0x9b, 0x76, 0x83, 0x47, 0x1b, 0xee, 0xd5, + 0x1b, 0xaa, 0xa7, 0x64, 0x97, 0xec, 0x02, 0x33, 0x17, 0xfa, 0xff, 0x25, 0x3c, 0xb2, 0xa8, 0x76, + 0xda, 0xb1, 0xce, 0x9a, 0x06, 0x3a, 0x57, 0x0c, 0xdc, 0xbf, 0x31, 0xf8, 0xec, 0xe8, 0x65, 0x0b, + 0x67, 0xfc, 0x93, 0x4e, 0xfa, 0x9f, 0x50, 0xba, 0xc1, 0x3e, 0x8b, 0x2a, 0x66, 0x9b, 0xd0, 0xa2, + 0x7d, 0x45, 0xe0, 0x93, 0xe1, 0xd6, 0xe6, 0x78, 0x08, 0x9b, 0xe6, 0x6f, 0xa1, 0x7d, 0x9c, 0x9d, + 0xe3, 0x89, 0x92, 0x8a, 0xee, 0x73, 0x90, 0x15, 0xaa, 0xe4, 0xa2, 0xcf, 0xe4, 0x07, 0x9d, 0x2c, + 0x25, 0x30, 0x95, 0x3d, 0xf0, 0xa2, 0xb0, 0x08, 0x3c, 0x1a, 0x7e, 0xb7, 0x49, 0x50, 0xe8, 0x44, + 0xd8, 0xfd, 0x3e, 0xeb, 0x39, 0x3f, 0x66, 0x3d, 0xe7, 0xe7, 0xac, 0xe7, 0x7c, 0xf9, 0xd5, 0xbb, + 0x73, 0xd6, 0xa2, 0xdf, 0xe5, 0xf3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe6, 0xcf, 0x20, 0xf1, + 0x3f, 0x05, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index ec060e355..6294b1091 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -28,12 +28,12 @@ message BlockDataRequest { } message BlockDataResponse { - repeated uint64 BitmapIDs = 1; - repeated uint64 ProfileIDs = 2; + repeated uint64 RowIDs = 1; + repeated uint64 ColumnIDs = 2; } message Cache { - repeated uint64 BitmapIDs = 1; + repeated uint64 IDs = 1; } message MaxSlicesResponse { diff --git a/internal/public.pb.go b/internal/public.pb.go index 249bbfddf..24fb7eb85 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -12,7 +12,7 @@ Bitmap Pair Bit - Profile + ColumnAttrSet Attr AttrMap QueryRequest @@ -67,8 +67,8 @@ func (*Pair) ProtoMessage() {} func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} } type Bit struct { - BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,proto3" json:"BitmapID,omitempty"` - ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,proto3" json:"ProfileID,omitempty"` + RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"` + ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"` Timestamp int64 `protobuf:"varint,3,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"` } @@ -77,17 +77,17 @@ func (m *Bit) String() string { return proto.CompactTextString(m) } func (*Bit) ProtoMessage() {} func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} } -type Profile struct { +type ColumnAttrSet struct { ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"` Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } -func (m *Profile) Reset() { *m = Profile{} } -func (m *Profile) String() string { return proto.CompactTextString(m) } -func (*Profile) ProtoMessage() {} -func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } +func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} } +func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) } +func (*ColumnAttrSet) ProtoMessage() {} +func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} } -func (m *Profile) GetAttrs() []*Attr { +func (m *ColumnAttrSet) GetAttrs() []*Attr { if m != nil { return m.Attrs } @@ -125,11 +125,11 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` - Profiles bool `protobuf:"varint,3,opt,name=Profiles,proto3" json:"Profiles,omitempty"` - Quantum string `protobuf:"bytes,4,opt,name=Quantum,proto3" json:"Quantum,omitempty"` - Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` + Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` + ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` + Quantum string `protobuf:"bytes,4,opt,name=Quantum,proto3" json:"Quantum,omitempty"` + Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } @@ -138,9 +138,9 @@ func (*QueryRequest) ProtoMessage() {} func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} } type QueryResponse struct { - Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` - Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` - Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"` + Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"` + Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"` + ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"` } func (m *QueryResponse) Reset() { *m = QueryResponse{} } @@ -155,9 +155,9 @@ func (m *QueryResponse) GetResults() []*QueryResult { return nil } -func (m *QueryResponse) GetProfiles() []*Profile { +func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet { if m != nil { - return m.Profiles + return m.ColumnAttrSets } return nil } @@ -192,8 +192,8 @@ type ImportRequest struct { DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"` Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"` Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"` - BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"` - ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"` + RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"` + ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"` Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"` } @@ -206,7 +206,7 @@ func init() { proto.RegisterType((*Bitmap)(nil), "internal.Bitmap") proto.RegisterType((*Pair)(nil), "internal.Pair") proto.RegisterType((*Bit)(nil), "internal.Bit") - proto.RegisterType((*Profile)(nil), "internal.Profile") + proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet") proto.RegisterType((*Attr)(nil), "internal.Attr") proto.RegisterType((*AttrMap)(nil), "internal.AttrMap") proto.RegisterType((*QueryRequest)(nil), "internal.QueryRequest") @@ -304,15 +304,15 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.BitmapID != 0 { + if m.RowID != 0 { dAtA[i] = 0x8 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.BitmapID)) + i = encodeVarintPublic(dAtA, i, uint64(m.RowID)) } - if m.ProfileID != 0 { + if m.ColumnID != 0 { dAtA[i] = 0x10 i++ - i = encodeVarintPublic(dAtA, i, uint64(m.ProfileID)) + i = encodeVarintPublic(dAtA, i, uint64(m.ColumnID)) } if m.Timestamp != 0 { dAtA[i] = 0x18 @@ -322,7 +322,7 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) { return i, nil } -func (m *Profile) Marshal() (dAtA []byte, err error) { +func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -332,7 +332,7 @@ func (m *Profile) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Profile) MarshalTo(dAtA []byte) (int, error) { +func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -480,10 +480,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(j3)) i += copy(dAtA[i:], dAtA4[:j3]) } - if m.Profiles { + if m.ColumnAttrs { dAtA[i] = 0x18 i++ - if m.Profiles { + if m.ColumnAttrs { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -542,8 +542,8 @@ func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) { i += n } } - if len(m.Profiles) > 0 { - for _, msg := range m.Profiles { + if len(m.ColumnAttrSets) > 0 { + for _, msg := range m.ColumnAttrSets { dAtA[i] = 0x1a i++ i = encodeVarintPublic(dAtA, i, uint64(msg.Size())) @@ -644,10 +644,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPublic(dAtA, i, uint64(m.Slice)) } - if len(m.BitmapIDs) > 0 { - dAtA7 := make([]byte, len(m.BitmapIDs)*10) + if len(m.RowIDs) > 0 { + dAtA7 := make([]byte, len(m.RowIDs)*10) var j6 int - for _, num := range m.BitmapIDs { + for _, num := range m.RowIDs { for num >= 1<<7 { dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -661,10 +661,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPublic(dAtA, i, uint64(j6)) i += copy(dAtA[i:], dAtA7[:j6]) } - if len(m.ProfileIDs) > 0 { - dAtA9 := make([]byte, len(m.ProfileIDs)*10) + if len(m.ColumnIDs) > 0 { + dAtA9 := make([]byte, len(m.ColumnIDs)*10) var j8 int - for _, num := range m.ProfileIDs { + for _, num := range m.ColumnIDs { for num >= 1<<7 { dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -760,11 +760,11 @@ func (m *Pair) Size() (n int) { func (m *Bit) Size() (n int) { var l int _ = l - if m.BitmapID != 0 { - n += 1 + sovPublic(uint64(m.BitmapID)) + if m.RowID != 0 { + n += 1 + sovPublic(uint64(m.RowID)) } - if m.ProfileID != 0 { - n += 1 + sovPublic(uint64(m.ProfileID)) + if m.ColumnID != 0 { + n += 1 + sovPublic(uint64(m.ColumnID)) } if m.Timestamp != 0 { n += 1 + sovPublic(uint64(m.Timestamp)) @@ -772,7 +772,7 @@ func (m *Bit) Size() (n int) { return n } -func (m *Profile) Size() (n int) { +func (m *ColumnAttrSet) Size() (n int) { var l int _ = l if m.ID != 0 { @@ -839,7 +839,7 @@ func (m *QueryRequest) Size() (n int) { } n += 1 + sovPublic(uint64(l)) + l } - if m.Profiles { + if m.ColumnAttrs { n += 2 } l = len(m.Quantum) @@ -865,8 +865,8 @@ func (m *QueryResponse) Size() (n int) { n += 1 + l + sovPublic(uint64(l)) } } - if len(m.Profiles) > 0 { - for _, e := range m.Profiles { + if len(m.ColumnAttrSets) > 0 { + for _, e := range m.ColumnAttrSets { l = e.Size() n += 1 + l + sovPublic(uint64(l)) } @@ -910,16 +910,16 @@ func (m *ImportRequest) Size() (n int) { if m.Slice != 0 { n += 1 + sovPublic(uint64(m.Slice)) } - if len(m.BitmapIDs) > 0 { + if len(m.RowIDs) > 0 { l = 0 - for _, e := range m.BitmapIDs { + for _, e := range m.RowIDs { l += sovPublic(uint64(e)) } n += 1 + sovPublic(uint64(l)) + l } - if len(m.ProfileIDs) > 0 { + if len(m.ColumnIDs) > 0 { l = 0 - for _, e := range m.ProfileIDs { + for _, e := range m.ColumnIDs { l += sovPublic(uint64(e)) } n += 1 + sovPublic(uint64(l)) + l @@ -1209,9 +1209,9 @@ func (m *Bit) Unmarshal(dAtA []byte) error { switch fieldNum { case 1: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field BitmapID", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RowID", wireType) } - m.BitmapID = 0 + m.RowID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1221,16 +1221,16 @@ func (m *Bit) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.BitmapID |= (uint64(b) & 0x7F) << shift + m.RowID |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } } case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ProfileID", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnID", wireType) } - m.ProfileID = 0 + m.ColumnID = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPublic @@ -1240,7 +1240,7 @@ func (m *Bit) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ProfileID |= (uint64(b) & 0x7F) << shift + m.ColumnID |= (uint64(b) & 0x7F) << shift if b < 0x80 { break } @@ -1285,7 +1285,7 @@ func (m *Bit) Unmarshal(dAtA []byte) error { } return nil } -func (m *Profile) Unmarshal(dAtA []byte) error { +func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -1308,10 +1308,10 @@ func (m *Profile) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Profile: wiretype end group for non-group") + return fmt.Errorf("proto: ColumnAttrSet: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Profile: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ColumnAttrSet: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -1772,7 +1772,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { } case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnAttrs", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -1789,7 +1789,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.Profiles = bool(v != 0) + m.ColumnAttrs = bool(v != 0) case 4: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Quantum", wireType) @@ -1951,7 +1951,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnAttrSets", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -1975,8 +1975,8 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Profiles = append(m.Profiles, &Profile{}) - if err := m.Profiles[len(m.Profiles)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.ColumnAttrSets = append(m.ColumnAttrSets, &ColumnAttrSet{}) + if err := m.ColumnAttrSets[len(m.ColumnAttrSets)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -2300,7 +2300,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.RowIDs = append(m.RowIDs, v) } } else if wireType == 0 { var v uint64 @@ -2318,9 +2318,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.BitmapIDs = append(m.BitmapIDs, v) + m.RowIDs = append(m.RowIDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType) } case 5: if wireType == 2 { @@ -2362,7 +2362,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.ProfileIDs = append(m.ProfileIDs, v) + m.ColumnIDs = append(m.ColumnIDs, v) } } else if wireType == 0 { var v uint64 @@ -2380,9 +2380,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { break } } - m.ProfileIDs = append(m.ProfileIDs, v) + m.ColumnIDs = append(m.ColumnIDs, v) } else { - return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType) } case 6: if wireType == 2 { @@ -2575,41 +2575,42 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 570 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x6e, 0xd4, 0x40, - 0x10, 0xa5, 0x6d, 0xcf, 0xaf, 0x26, 0x19, 0x0d, 0x2d, 0x40, 0x16, 0x42, 0x23, 0xcb, 0x62, 0xe1, - 0x0d, 0x13, 0x29, 0x1c, 0x00, 0xe1, 0x4c, 0x22, 0x8d, 0x10, 0x51, 0xd2, 0x89, 0xd8, 0xb1, 0x70, - 0x42, 0x13, 0x2c, 0xf9, 0x47, 0x77, 0x7b, 0x31, 0x4b, 0x16, 0x6c, 0x38, 0x01, 0x47, 0x80, 0x9b, - 0xb0, 0xe4, 0x08, 0x68, 0xb8, 0x08, 0xaa, 0xfe, 0xd8, 0x66, 0x83, 0xd8, 0xf5, 0x7b, 0xe5, 0xea, - 0xae, 0xf7, 0xaa, 0xca, 0x70, 0xd0, 0xb4, 0x37, 0x45, 0x7e, 0xbb, 0x6e, 0x44, 0xad, 0x6a, 0x3a, - 0xcd, 0x2b, 0xc5, 0x45, 0x95, 0x15, 0x71, 0x0a, 0xe3, 0x34, 0x57, 0x65, 0xd6, 0x50, 0x0a, 0x41, - 0x9a, 0x2b, 0x19, 0x92, 0xc8, 0x4f, 0x02, 0xa6, 0xcf, 0xf4, 0x29, 0x8c, 0x5e, 0x2a, 0x25, 0x64, - 0xe8, 0x45, 0x7e, 0x32, 0x3f, 0x5e, 0xac, 0x5d, 0xde, 0x1a, 0x69, 0x66, 0x82, 0xf1, 0x1a, 0x82, - 0x8b, 0x2c, 0x17, 0x74, 0x09, 0xfe, 0x2b, 0xbe, 0x0b, 0x49, 0x44, 0x92, 0x80, 0xe1, 0x91, 0x3e, - 0x80, 0xd1, 0x49, 0xdd, 0x56, 0x2a, 0xf4, 0x34, 0x67, 0x40, 0xfc, 0x16, 0xfc, 0x34, 0x57, 0xf4, - 0x31, 0x4c, 0xcd, 0xd3, 0xdb, 0x8d, 0xcd, 0xe9, 0x30, 0x7d, 0x02, 0xb3, 0x0b, 0x51, 0xbf, 0xcf, - 0x0b, 0xbe, 0xdd, 0xd8, 0xe4, 0x9e, 0xc0, 0xe8, 0x75, 0x5e, 0x72, 0xa9, 0xb2, 0xb2, 0x09, 0xfd, - 0x88, 0x24, 0x3e, 0xeb, 0x89, 0xf8, 0x05, 0x4c, 0xec, 0xa7, 0x74, 0x01, 0x5e, 0x77, 0xb9, 0xb7, - 0xdd, 0xfc, 0xa7, 0x9e, 0x6f, 0x04, 0x02, 0x3c, 0x0d, 0x05, 0xcd, 0x8c, 0x20, 0x0a, 0xc1, 0xf5, - 0xae, 0xe1, 0xb6, 0x24, 0x7d, 0xa6, 0x11, 0xcc, 0xaf, 0x94, 0xc8, 0xab, 0xbb, 0x37, 0x59, 0xd1, - 0x72, 0x5d, 0xcf, 0x8c, 0x0d, 0x29, 0x54, 0xba, 0xad, 0x94, 0x09, 0x07, 0xba, 0xdc, 0x0e, 0xa3, - 0x96, 0xb4, 0xae, 0x0b, 0x13, 0x1c, 0x45, 0x24, 0x99, 0xb2, 0x9e, 0xa0, 0x2b, 0x80, 0xb3, 0xa2, - 0xce, 0x6c, 0xee, 0x38, 0x22, 0x09, 0x61, 0x03, 0x26, 0x3e, 0x82, 0x09, 0x56, 0xfa, 0x3a, 0x6b, - 0x7a, 0x6d, 0xe4, 0x5f, 0xda, 0xbe, 0x10, 0x38, 0xb8, 0x6c, 0xb9, 0xd8, 0x31, 0xfe, 0xb1, 0xe5, - 0x52, 0x61, 0x8b, 0x34, 0xb6, 0x2a, 0x0d, 0xa0, 0x8f, 0x60, 0x7c, 0x55, 0xe4, 0xb7, 0xdc, 0x38, - 0x15, 0x30, 0x8b, 0x50, 0x89, 0xf5, 0x56, 0x6a, 0xa1, 0x53, 0xd6, 0x61, 0x1a, 0xc2, 0xe4, 0xb2, - 0xcd, 0x2a, 0xd5, 0x96, 0x5a, 0xe4, 0x8c, 0x39, 0x88, 0xb7, 0x31, 0x5e, 0xd6, 0xca, 0x09, 0xb4, - 0x28, 0xfe, 0x44, 0xe0, 0xd0, 0x16, 0x23, 0x9b, 0xba, 0x92, 0x1c, 0x1d, 0x3f, 0x15, 0xc2, 0x39, - 0x7e, 0x2a, 0x04, 0x3d, 0x82, 0x09, 0xe3, 0xb2, 0x2d, 0x94, 0x6b, 0xda, 0xc3, 0x5e, 0x98, 0xcb, - 0x6d, 0x0b, 0xc5, 0xdc, 0x57, 0xf4, 0xd9, 0x5f, 0x25, 0x62, 0xc6, 0xfd, 0x3e, 0xc3, 0x46, 0xfa, - 0xaa, 0xe3, 0xcf, 0x04, 0xe6, 0x83, 0x7b, 0x68, 0xe2, 0x16, 0x42, 0x17, 0x31, 0x3f, 0x5e, 0xf6, - 0xc9, 0x86, 0x67, 0x6e, 0x61, 0x0e, 0x80, 0x9c, 0xdb, 0x41, 0x20, 0xe7, 0x68, 0x3f, 0x2e, 0x81, - 0x7b, 0x73, 0x60, 0x3f, 0xd2, 0xcc, 0x04, 0xd1, 0xa3, 0x93, 0x0f, 0x59, 0x75, 0xc7, 0xdf, 0x69, - 0x8f, 0xa6, 0xcc, 0xc1, 0xf8, 0x3b, 0x81, 0xc3, 0x6d, 0xd9, 0xd4, 0x42, 0xb9, 0xce, 0x2c, 0xc0, - 0xdb, 0xa4, 0xd6, 0x0a, 0x6f, 0x93, 0x62, 0xa7, 0xce, 0x44, 0x56, 0x9a, 0xe1, 0x9b, 0x31, 0x03, - 0x90, 0xd5, 0xbd, 0xd1, 0xed, 0x08, 0x98, 0x01, 0x7a, 0xaa, 0xec, 0x2e, 0xc9, 0x30, 0xd0, 0x2d, - 0xec, 0x09, 0x9c, 0xaa, 0x6e, 0x99, 0x64, 0x38, 0xd2, 0xe1, 0x01, 0x83, 0xf1, 0x6e, 0x9d, 0x64, - 0x38, 0x8e, 0xfc, 0xc4, 0x67, 0x03, 0x26, 0x5d, 0xfe, 0xd8, 0xaf, 0xc8, 0xcf, 0xfd, 0x8a, 0xfc, - 0xda, 0xaf, 0xc8, 0xd7, 0xdf, 0xab, 0x7b, 0x37, 0x63, 0xfd, 0x5f, 0x79, 0xfe, 0x27, 0x00, 0x00, - 0xff, 0xff, 0x37, 0xb6, 0x15, 0x22, 0x67, 0x04, 0x00, 0x00, + // 579 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0x66, 0x6d, 0x27, 0x4d, 0x26, 0x6d, 0x14, 0xad, 0xf8, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x83, + 0x4f, 0xa9, 0x54, 0x1e, 0x00, 0xe1, 0x24, 0x95, 0x22, 0x44, 0x45, 0x27, 0x85, 0xbb, 0x5b, 0x56, + 0xc5, 0x92, 0xff, 0x58, 0xaf, 0x85, 0xf2, 0x00, 0xdc, 0x91, 0xb8, 0x70, 0xe5, 0xc6, 0xa3, 0x70, + 0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0xbb, 0xde, 0xd8, 0xe5, 0x80, 0xb8, 0xed, 0xf7, 0xcd, 0xce, + 0x7a, 0xbe, 0xf9, 0x66, 0x0c, 0xc7, 0x55, 0x73, 0x9d, 0xa5, 0x37, 0x8b, 0x4a, 0x96, 0xaa, 0xe4, + 0xa3, 0xb4, 0x50, 0x42, 0x16, 0x49, 0x16, 0xc6, 0x30, 0x8c, 0x53, 0x95, 0x27, 0x15, 0xe7, 0xe0, + 0xc5, 0xa9, 0xaa, 0x7d, 0x16, 0xb8, 0x91, 0x87, 0xfa, 0xcc, 0x9f, 0xc2, 0xe0, 0x85, 0x52, 0xb2, + 0xf6, 0x9d, 0xc0, 0x8d, 0x26, 0x67, 0xd3, 0x85, 0xcd, 0x5b, 0x10, 0x8d, 0x26, 0x18, 0x2e, 0xc0, + 0x7b, 0x9d, 0xa4, 0x92, 0xcf, 0xc0, 0x7d, 0x29, 0x76, 0x3e, 0x0b, 0x58, 0xe4, 0x21, 0x1d, 0xf9, + 0x7d, 0x18, 0x2c, 0xcb, 0xa6, 0x50, 0xbe, 0xa3, 0x39, 0x03, 0xc2, 0x37, 0xe0, 0xc6, 0xa9, 0xa2, + 0x20, 0x96, 0x1f, 0x37, 0xab, 0x36, 0xc1, 0x00, 0xfe, 0x18, 0x46, 0xcb, 0x32, 0x6b, 0xf2, 0x62, + 0xb3, 0x6a, 0xb3, 0x0e, 0x98, 0x3f, 0x81, 0xf1, 0x55, 0x9a, 0x8b, 0x5a, 0x25, 0x79, 0xe5, 0xbb, + 0x01, 0x8b, 0x5c, 0xec, 0x88, 0x70, 0x0d, 0x27, 0xe6, 0x26, 0x55, 0xb5, 0x15, 0x8a, 0x4f, 0xc1, + 0x39, 0xbc, 0xee, 0x6c, 0x56, 0xff, 0xa9, 0xe6, 0x3b, 0x03, 0x8f, 0x4e, 0x7d, 0x39, 0x63, 0x23, + 0x87, 0x83, 0x77, 0xb5, 0xab, 0x44, 0x5b, 0x97, 0x3e, 0xf3, 0x00, 0x26, 0x5b, 0x25, 0xd3, 0xe2, + 0xf6, 0x6d, 0x92, 0x35, 0x42, 0x57, 0x35, 0xc6, 0x3e, 0x45, 0x8a, 0x36, 0x85, 0x32, 0x61, 0x4f, + 0x17, 0x7d, 0xc0, 0xa4, 0x28, 0x2e, 0xcb, 0xcc, 0x04, 0x07, 0x01, 0x8b, 0x46, 0xd8, 0x11, 0x7c, + 0x0e, 0x70, 0x9e, 0x95, 0x49, 0x9b, 0x3b, 0x0c, 0x58, 0xc4, 0xb0, 0xc7, 0x84, 0xa7, 0x70, 0x44, + 0x95, 0xbe, 0x4a, 0xaa, 0x4e, 0x1b, 0xfb, 0x97, 0xb6, 0xcf, 0x0c, 0x8e, 0x2f, 0x1b, 0x21, 0x77, + 0x28, 0x3e, 0x34, 0xa2, 0xd6, 0x1e, 0x68, 0xdc, 0xaa, 0x34, 0x80, 0x3f, 0x84, 0xe1, 0x36, 0x4b, + 0x6f, 0x84, 0xe9, 0x94, 0x87, 0x2d, 0x22, 0xad, 0x5d, 0x87, 0x6b, 0xad, 0x75, 0x84, 0x7d, 0x8a, + 0xfb, 0x70, 0x74, 0xd9, 0x24, 0x85, 0x6a, 0x72, 0x2d, 0x75, 0x8c, 0x16, 0xd2, 0x9b, 0x28, 0xf2, + 0x52, 0x59, 0x99, 0x2d, 0x0a, 0xbf, 0x30, 0x38, 0x69, 0x4b, 0xaa, 0xab, 0xb2, 0xa8, 0x05, 0xf5, + 0x7d, 0x2d, 0xa5, 0xed, 0xfb, 0x5a, 0x4a, 0x7e, 0x0a, 0x47, 0x28, 0xea, 0x26, 0x53, 0xd6, 0xba, + 0x07, 0x9d, 0x3c, 0x9b, 0xdb, 0x64, 0x0a, 0xed, 0x2d, 0xfe, 0x1c, 0xa6, 0x77, 0x46, 0x81, 0x6a, + 0xa5, 0xbc, 0x47, 0x5d, 0xde, 0x9d, 0x38, 0xfe, 0x75, 0x3d, 0xfc, 0xc4, 0x60, 0xd2, 0x7b, 0x99, + 0x47, 0x76, 0x4d, 0x74, 0x59, 0x93, 0xb3, 0x59, 0xf7, 0x90, 0xe1, 0xd1, 0xae, 0xd1, 0x31, 0xb0, + 0x8b, 0x76, 0x40, 0xd8, 0x05, 0xd9, 0x42, 0xab, 0x61, 0xbf, 0xdf, 0xb3, 0x85, 0x68, 0x34, 0x41, + 0xea, 0xda, 0xf2, 0x7d, 0x52, 0xdc, 0x8a, 0x77, 0xba, 0x6b, 0x23, 0xb4, 0x30, 0xfc, 0xc6, 0xe0, + 0x64, 0x93, 0x57, 0xa5, 0x54, 0xd6, 0xb1, 0x29, 0x38, 0xab, 0xb8, 0x6d, 0x8e, 0xb3, 0x8a, 0xc9, + 0xc1, 0x73, 0x99, 0xe4, 0x66, 0x28, 0xc7, 0x68, 0x00, 0xb1, 0xda, 0x33, 0xed, 0x91, 0x87, 0x06, + 0x68, 0x0f, 0x68, 0xc9, 0x6a, 0xdf, 0x33, 0xbe, 0x1a, 0x44, 0x53, 0x68, 0x77, 0xac, 0xf6, 0x07, + 0x3a, 0xd4, 0x11, 0x34, 0x85, 0x87, 0x25, 0xab, 0xfd, 0x61, 0xe0, 0x46, 0x2e, 0xf6, 0x98, 0x78, + 0xf6, 0x63, 0x3f, 0x67, 0x3f, 0xf7, 0x73, 0xf6, 0x6b, 0x3f, 0x67, 0x5f, 0x7f, 0xcf, 0xef, 0x5d, + 0x0f, 0xf5, 0x5f, 0xe6, 0xd9, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xaf, 0x79, 0x70, 0xf4, 0x75, + 0x04, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index 03cfdaabd..91c1f31e8 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -13,12 +13,12 @@ message Pair { } message Bit { - uint64 BitmapID = 1; - uint64 ProfileID = 2; + uint64 RowID = 1; + uint64 ColumnID = 2; int64 Timestamp = 3; } -message Profile { +message ColumnAttrSet { uint64 ID = 1; repeated Attr Attrs = 2; } @@ -39,7 +39,7 @@ message AttrMap { message QueryRequest { string Query = 1; repeated uint64 Slices = 2; - bool Profiles = 3; + bool ColumnAttrs = 3; string Quantum = 4; bool Remote = 5; } @@ -47,7 +47,7 @@ message QueryRequest { message QueryResponse { string Err = 1; repeated QueryResult Results = 2; - repeated Profile Profiles = 3; + repeated ColumnAttrSet ColumnAttrSets = 3; } message QueryResult { @@ -61,7 +61,7 @@ message ImportRequest { string DB = 1; string Frame = 2; uint64 Slice = 3; - repeated uint64 BitmapIDs = 4; - repeated uint64 ProfileIDs = 5; + repeated uint64 RowIDs = 4; + repeated uint64 ColumnIDs = 5; repeated int64 Timestamps = 6; } diff --git a/iterator.go b/iterator.go index 282cb32ed..155914491 100644 --- a/iterator.go +++ b/iterator.go @@ -6,19 +6,19 @@ import ( "github.com/pilosa/pilosa/roaring" ) -// Iterator is an interface for looping over bitmap/profile pairs. +// Iterator is an interface for looping over row/column pairs. type Iterator interface { - Seek(bitmapID, profileID uint64) - Next() (bitmapID, profileID uint64, eof bool) + Seek(rowID, columnID uint64) + Next() (rowID, columnID uint64, eof bool) } // BufIterator wraps an iterator to provide the ability to unread values. type BufIterator struct { buf struct { - bitmapID uint64 - profileID uint64 - eof bool - full bool + rowID uint64 + columnID uint64 + eof bool + full bool } itr Iterator } @@ -29,28 +29,28 @@ func NewBufIterator(itr Iterator) *BufIterator { } // Seek moves to the first pair equal to or greater than pseek/bseek. -func (itr *BufIterator) Seek(bitmapID, profileID uint64) { +func (itr *BufIterator) Seek(rowID, columnID uint64) { itr.buf.full = false - itr.itr.Seek(bitmapID, profileID) + itr.itr.Seek(rowID, columnID) } -// Next returns the next pair in the bitmap. +// Next returns the next pair in the row. // If a value has been buffered then it is returned and the buffer is cleared. -func (itr *BufIterator) Next() (bitmapID, profileID uint64, eof bool) { +func (itr *BufIterator) Next() (rowID, columnID uint64, eof bool) { if itr.buf.full { itr.buf.full = false - return itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof + return itr.buf.rowID, itr.buf.columnID, itr.buf.eof } // Read values onto buffer in case of unread. - itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof = itr.itr.Next() + itr.buf.rowID, itr.buf.columnID, itr.buf.eof = itr.itr.Next() - return itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof + return itr.buf.rowID, itr.buf.columnID, itr.buf.eof } // Peek reads the next value but leaves it on the buffer. -func (itr *BufIterator) Peek() (bitmapID, profileID uint64, eof bool) { - bitmapID, profileID, eof = itr.Next() +func (itr *BufIterator) Peek() (rowID, columnID uint64, eof bool) { + rowID, columnID, eof = itr.Next() itr.Unread() return } @@ -64,30 +64,30 @@ func (itr *BufIterator) Unread() { itr.buf.full = true } -// LimitIterator wraps an Iterator and limits it to a max profile/bitmap pair. +// LimitIterator wraps an Iterator and limits it to a max column/row pair. type LimitIterator struct { - itr Iterator - maxBitmapID uint64 - maxProfileID uint64 + itr Iterator + maxRowID uint64 + maxColumnID uint64 eof bool } // NewLimitIterator returns a new LimitIterator. -func NewLimitIterator(itr Iterator, maxBitmapID, maxProfileID uint64) *LimitIterator { +func NewLimitIterator(itr Iterator, maxRowID, maxColumnID uint64) *LimitIterator { return &LimitIterator{ - itr: itr, - maxBitmapID: maxBitmapID, - maxProfileID: maxProfileID, + itr: itr, + maxRowID: maxRowID, + maxColumnID: maxColumnID, } } -// Seek moves the underlying iterator to a profile/bitmap pair. -func (itr *LimitIterator) Seek(bitmapID, profileID uint64) { itr.itr.Seek(bitmapID, profileID) } +// Seek moves the underlying iterator to a column/row pair. +func (itr *LimitIterator) Seek(rowID, columnID uint64) { itr.itr.Seek(rowID, columnID) } -// Next returns the next bitmap/profile ID pair. +// Next returns the next row/column ID pair. // If the underlying iterator returns a pair higher than the max then EOF is returned. -func (itr *LimitIterator) Next() (bitmapID, profileID uint64, eof bool) { +func (itr *LimitIterator) Next() (rowID, columnID uint64, eof bool) { // Always return EOF once it is reached by limit or the underlying iterator. if itr.eof { return 0, 0, true @@ -95,35 +95,35 @@ func (itr *LimitIterator) Next() (bitmapID, profileID uint64, eof bool) { // Retrieve pair from underlying iterator. // Mark as EOF if it is beyond the limit (or at EOF). - bitmapID, profileID, eof = itr.itr.Next() - if eof || bitmapID > itr.maxBitmapID || (bitmapID == itr.maxBitmapID && profileID > itr.maxProfileID) { + rowID, columnID, eof = itr.itr.Next() + if eof || rowID > itr.maxRowID || (rowID == itr.maxRowID && columnID > itr.maxColumnID) { itr.eof = true return 0, 0, true } - return bitmapID, profileID, false + return rowID, columnID, false } -// SliceIterator iterates over a pair of bitmap/profile ID slices. +// SliceIterator iterates over a pair of row/column ID slices. type SliceIterator struct { - bitmapIDs []uint64 - profileIDs []uint64 + rowIDs []uint64 + columnIDs []uint64 i, n int } -// NewSliceIterator returns an iterator to iterate over a set of bitmap/profile ID pairs. +// NewSliceIterator returns an iterator to iterate over a set of row/column ID pairs. // Both slices MUST have an equal length. Otherwise the function will panic. -func NewSliceIterator(bitmapIDs, profileIDs []uint64) *SliceIterator { - if len(profileIDs) != len(bitmapIDs) { - panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(bitmapIDs), len(profileIDs))) +func NewSliceIterator(rowIDs, columnIDs []uint64) *SliceIterator { + if len(columnIDs) != len(rowIDs) { + panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(rowIDs), len(columnIDs))) } return &SliceIterator{ - bitmapIDs: bitmapIDs, - profileIDs: profileIDs, + rowIDs: rowIDs, + columnIDs: columnIDs, - n: len(bitmapIDs), + n: len(rowIDs), } } @@ -131,10 +131,10 @@ func NewSliceIterator(bitmapIDs, profileIDs []uint64) *SliceIterator { // If the pair is not found, the iterator seeks to the next pair. func (itr *SliceIterator) Seek(bseek, pseek uint64) { for i := 0; i < itr.n; i++ { - bitmapID := itr.bitmapIDs[i] - profileID := itr.profileIDs[i] + rowID := itr.rowIDs[i] + columnID := itr.columnIDs[i] - if (bseek == bitmapID && pseek <= profileID) || bseek < bitmapID { + if (bseek == rowID && pseek <= columnID) || bseek < rowID { itr.i = i return } @@ -144,20 +144,20 @@ func (itr *SliceIterator) Seek(bseek, pseek uint64) { itr.i = itr.n } -// Next returns the next bitmap/profile ID pair. -func (itr *SliceIterator) Next() (bitmapID, profileID uint64, eof bool) { +// Next returns the next row/column ID pair. +func (itr *SliceIterator) Next() (rowID, columnID uint64, eof bool) { if itr.i >= itr.n { return 0, 0, true } - bitmapID = itr.bitmapIDs[itr.i] - profileID = itr.profileIDs[itr.i] + rowID = itr.rowIDs[itr.i] + columnID = itr.columnIDs[itr.i] itr.i++ - return bitmapID, profileID, false + return rowID, columnID, false } -// RoaringIterator converts a roaring.Iterator to output profile/bitmap pairs. +// RoaringIterator converts a roaring.Iterator to output column/row pairs. type RoaringIterator struct { itr *roaring.Iterator } @@ -173,8 +173,8 @@ func (itr *RoaringIterator) Seek(bseek, pseek uint64) { itr.itr.Seek((bseek * SliceWidth) + pseek) } -// Next returns the next profile/bitmap ID pair. -func (itr *RoaringIterator) Next() (bitmapID, profileID uint64, eof bool) { +// Next returns the next column/row ID pair. +func (itr *RoaringIterator) Next() (rowID, columnID uint64, eof bool) { v, eof := itr.itr.Next() return v / SliceWidth, v % SliceWidth, eof } diff --git a/pilosa.go b/pilosa.go index b754c0e48..8ffb54155 100644 --- a/pilosa.go +++ b/pilosa.go @@ -35,54 +35,54 @@ var ( // Todo: remove . when frame doesn't require . for topN var nameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,64}$`) -// Profile represents vertical column in a database. -// A profile can have a set of attributes attached to it. -type Profile struct { +// ColumnAttrSet represents a set of attributes for a vertical column in a database. +// Can have a set of attributes attached to it. +type ColumnAttrSet struct { ID uint64 `json:"id"` Attrs map[string]interface{} `json:"attrs,omitempty"` } -// encodeProfiles converts a into its internal representation. -func encodeProfiles(a []*Profile) []*internal.Profile { - other := make([]*internal.Profile, len(a)) +// encodeColumnAttrSets converts a into its internal representation. +func encodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet { + other := make([]*internal.ColumnAttrSet, len(a)) for i := range a { - other[i] = encodeProfile(a[i]) + other[i] = encodeColumnAttrSet(a[i]) } return other } -// decodeProfiles converts a from its internal representation. -func decodeProfiles(a []*internal.Profile) []*Profile { - other := make([]*Profile, len(a)) +// decodeColumnAttrSets converts a from its internal representation. +func decodeColumnAttrSets(a []*internal.ColumnAttrSet) []*ColumnAttrSet { + other := make([]*ColumnAttrSet, len(a)) for i := range a { - other[i] = decodeProfile(a[i]) + other[i] = decodeColumnAttrSet(a[i]) } return other } -// encodeProfile converts p into its internal representation. -func encodeProfile(p *Profile) *internal.Profile { - return &internal.Profile{ - ID: p.ID, - Attrs: encodeAttrs(p.Attrs), +// encodeColumnAttrSet converts set into its internal representation. +func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet { + return &internal.ColumnAttrSet{ + ID: set.ID, + Attrs: encodeAttrs(set.Attrs), } } -// decodeProfile converts b from its internal representation. -func decodeProfile(pb *internal.Profile) *Profile { - p := &Profile{ +// decodeColumnAttrSet converts b from its internal representation. +func decodeColumnAttrSet(pb *internal.ColumnAttrSet) *ColumnAttrSet { + set := &ColumnAttrSet{ ID: pb.ID, } if len(pb.Attrs) > 0 { - p.Attrs = make(map[string]interface{}, len(pb.Attrs)) + set.Attrs = make(map[string]interface{}, len(pb.Attrs)) for _, attr := range pb.Attrs { k, v := decodeAttr(attr) - p.Attrs[k] = v + set.Attrs[k] = v } } - return p + return set } // TimeFormat is the go-style time format used to parse string dates. diff --git a/server/server_test.go b/server/server_test.go index 8cd7c1d24..0f8a2f5d9 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -45,18 +45,18 @@ func TestMain_Set_Quick(t *testing.T) { if err := client.CreateFrame(context.Background(), "d", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists { t.Fatal(err) } - if _, err := m.Query("d", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil { + if _, err := m.Query("d", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil { t.Fatal(err) } } // Validate data. for frame, frameSet := range SetCommands(cmds).Frames() { - for id, profileIDs := range frameSet { + for id, columnIDs := range frameSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "bits": profileIDs, + "bits": columnIDs, "attrs": map[string]interface{}{}, }, }, @@ -75,11 +75,11 @@ func TestMain_Set_Quick(t *testing.T) { // Validate data after reopening. for frame, frameSet := range SetCommands(cmds).Frames() { - for id, profileIDs := range frameSet { + for id, columnIDs := range frameSet { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "bits": profileIDs, + "bits": columnIDs, "attrs": map[string]interface{}{}, }, }, @@ -102,8 +102,8 @@ func TestMain_Set_Quick(t *testing.T) { } } -// Ensure program can set bitmap attributes and retrieve them. -func TestMain_SetBitmapAttrs(t *testing.T) { +// Ensure program can set row attributes and retrieve them. +func TestMain_SetRowAttrs(t *testing.T) { m := MustRunMain() defer m.Close() @@ -119,36 +119,36 @@ func TestMain_SetBitmapAttrs(t *testing.T) { t.Fatal(err) } - // Set bits on different bitmaps in different frames. - if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil { + // Set bits on different rows in different frames. + if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=2, frame="x.n", profileID=100)`); err != nil { + } else if _, err := m.Query("d", "", `SetBit(id=2, frame="x.n", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=2, frame="z", profileID=100)`); err != nil { + } else if _, err := m.Query("d", "", `SetBit(id=2, frame="z", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=3, frame="neg", profileID=100)`); err != nil { + } else if _, err := m.Query("d", "", `SetBit(id=3, frame="neg", columnID=100)`); err != nil { t.Fatal(err) } - // Set bitmap attributes. - if _, err := m.Query("d", "", `SetBitmapAttrs(id=1, frame="x.n", x=100)`); err != nil { + // Set row attributes. + if _, err := m.Query("d", "", `SetRowAttrs(id=1, frame="x.n", x=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBitmapAttrs(id=2, frame="x.n", x=-200)`); err != nil { + } else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="x.n", x=-200)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBitmapAttrs(id=2, frame="z", x=300)`); err != nil { + } else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="z", x=300)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBitmapAttrs(id=3, frame="neg", x=-0.44)`); err != nil { + } else if _, err := m.Query("d", "", `SetRowAttrs(id=3, frame="neg", x=-0.44)`); err != nil { t.Fatal(err) } - // Query bitmap x.n/1. + // Query row x.n/1. if res, err := m.Query("d", "", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } - // Query bitmap x.n/2. + // Query row x.n/2. if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { @@ -159,19 +159,19 @@ func TestMain_SetBitmapAttrs(t *testing.T) { t.Fatal(err) } - // Query bitmaps after reopening. - if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { + // Query rows after reopening. + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - if res, err := m.Query("d", "profiles=true", `Bitmap(id=3, frame="neg")`); err != nil { + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=3, frame="neg")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } - // Query bitmap x.n/2. + // Query row x.n/2. if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { @@ -179,8 +179,8 @@ func TestMain_SetBitmapAttrs(t *testing.T) { } } -// Ensure program can set profile attributes and retrieve them. -func TestMain_SetProfileAttrs(t *testing.T) { +// Ensure program can set column attributes and retrieve them. +func TestMain_SetColumnAttrs(t *testing.T) { m := MustRunMain() defer m.Close() @@ -192,22 +192,22 @@ func TestMain_SetProfileAttrs(t *testing.T) { t.Fatal(err) } - // Set bits on bitmap. - if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil { + // Set bits on row. + if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil { t.Fatal(err) - } else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=101)`); err != nil { + } else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=101)`); err != nil { t.Fatal(err) } - // Set profile attributes. - if _, err := m.Query("d", "", `SetProfileAttrs(id=100, foo="bar")`); err != nil { + // Set column attributes. + if _, err := m.Query("d", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil { t.Fatal(err) } - // Query bitmap. - if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { + // Query row. + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -215,16 +215,16 @@ func TestMain_SetProfileAttrs(t *testing.T) { t.Fatal(err) } - // Query bitmap after reopening. - if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { + // Query row after reopening. + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } } -// Ensure program can set profile attributes with columnLabel option. -func TestMain_SetProfileAttrsWithColumnOption(t *testing.T) { +// Ensure program can set column attributes with columnLabel option. +func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) { m := MustRunMain() defer m.Close() @@ -236,22 +236,22 @@ func TestMain_SetProfileAttrsWithColumnOption(t *testing.T) { t.Fatal(err) } - // Set bits on bitmap. + // Set bits on row. if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil { t.Fatal(err) } else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil { t.Fatal(err) } - // Set profile attributes. - if _, err := m.Query("d", "", `SetProfileAttrs(col=100, foo="bar")`); err != nil { + // Set column attributes. + if _, err := m.Query("d", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil { t.Fatal(err) } - // Query bitmap. - if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil { + // Query row. + if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -282,18 +282,18 @@ func TestMain_FrameRestore(t *testing.T) { // Write data on first cluster. if _, err := m0.Query("d", "", ` - SetBit(id=1, frame="f", profileID=100) - SetBit(id=1, frame="f", profileID=1000) - SetBit(id=1, frame="f", profileID=100000) - SetBit(id=1, frame="f", profileID=200000) - SetBit(id=1, frame="f", profileID=400000) - SetBit(id=1, frame="f", profileID=600000) - SetBit(id=1, frame="f", profileID=800000) + SetBit(id=1, frame="f", columnID=100) + SetBit(id=1, frame="f", columnID=1000) + SetBit(id=1, frame="f", columnID=100000) + SetBit(id=1, frame="f", columnID=200000) + SetBit(id=1, frame="f", columnID=400000) + SetBit(id=1, frame="f", columnID=600000) + SetBit(id=1, frame="f", columnID=800000) `); err != nil { t.Fatal(err) } - // Query bitmap on first cluster. + // Query row on first cluster. if res, err := m0.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { @@ -316,7 +316,7 @@ func TestMain_FrameRestore(t *testing.T) { t.Fatal(err) } - // Query bitmap on second cluster. + // Query row on second cluster. if res, err := m2.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil { t.Fatal(err) } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { @@ -441,14 +441,14 @@ func (m *Main) Query(db, rawQuery, query string) (string, error) { // SetCommand represents a command to set a bit. type SetCommand struct { - ID uint64 - Frame string - ProfileID uint64 + ID uint64 + Frame string + ColumnID uint64 } type SetCommands []SetCommand -// Frames returns the set of profile ids for each frame/bitmap. +// Frames returns the set of column ids for each frame/row. func (a SetCommands) Frames() map[string]map[uint64][]uint64 { // Create a set of unique commands. m := make(map[SetCommand]struct{}) @@ -456,16 +456,16 @@ func (a SetCommands) Frames() map[string]map[uint64][]uint64 { m[cmd] = struct{}{} } - // Build unique ids for each frame & bitmap. + // Build unique ids for each frame & row. frames := make(map[string]map[uint64][]uint64) for cmd := range m { if frames[cmd.Frame] == nil { frames[cmd.Frame] = make(map[uint64][]uint64) } - frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ProfileID) + frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ColumnID) } - // Sort each set of profile ids. + // Sort each set of column ids. for _, frame := range frames { for id := range frame { sort.Sort(uint64Slice(frame[id])) @@ -480,9 +480,9 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { cmds := make([]SetCommand, rand.Intn(n)) for i := range cmds { cmds[i] = SetCommand{ - ID: uint64(rand.Intn(1000)), - Frame: "x.n", - ProfileID: uint64(rand.Intn(10)), + ID: uint64(rand.Intn(1000)), + Frame: "x.n", + ColumnID: uint64(rand.Intn(10)), } } return cmds diff --git a/view.go b/view.go index c7ec92733..4fa096943 100644 --- a/view.go +++ b/view.go @@ -38,8 +38,8 @@ type View struct { stats StatsClient - BitmapAttrStore *AttrStore - LogOutput io.Writer + RowAttrStore *AttrStore + LogOutput io.Writer } // NewView returns a new instance of View. @@ -124,7 +124,7 @@ func (v *View) openFragments() error { if err := frag.Open(); err != nil { return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err) } - frag.BitmapAttrStore = v.BitmapAttrStore + frag.RowAttrStore = v.RowAttrStore v.fragments[frag.Slice()] = frag v.stats.Count("maxSlice", 1) @@ -205,7 +205,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { if err := frag.Open(); err != nil { return nil, err } - frag.BitmapAttrStore = v.BitmapAttrStore + frag.RowAttrStore = v.RowAttrStore // Save to lookup. v.fragments[slice] = frag @@ -225,23 +225,23 @@ func (v *View) newFragment(path string, slice uint64) *Fragment { } // SetBit sets a bit within the view. -func (v *View) SetBit(bitmapID, profileID uint64) (changed bool, err error) { - slice := profileID / SliceWidth +func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) { + slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.SetBit(bitmapID, profileID) + return frag.SetBit(rowID, columnID) } // ClearBit clears a bit within the view. -func (v *View) ClearBit(bitmapID, profileID uint64) (changed bool, err error) { - slice := profileID / SliceWidth +func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) { + slice := columnID / SliceWidth frag, err := v.CreateFragmentIfNotExists(slice) if err != nil { return changed, err } - return frag.ClearBit(bitmapID, profileID) + return frag.ClearBit(rowID, columnID) } // IsInverseView returns true if the view is used for storing an inverted representation. diff --git a/view_test.go b/view_test.go index d10c90936..c531e1fce 100644 --- a/view_test.go +++ b/view_test.go @@ -10,7 +10,7 @@ import ( // View is a test wrapper for pilosa.View. type View struct { *pilosa.View - BitmapAttrStore *AttrStore + RowAttrStore *AttrStore } // NewView returns a new instance of View with a temporary path. @@ -22,10 +22,10 @@ func NewView(db, frame, name string) *View { file.Close() v := &View{ - View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize), - BitmapAttrStore: MustOpenAttrStore(), + View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize), + RowAttrStore: MustOpenAttrStore(), } - v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore + v.View.RowAttrStore = v.RowAttrStore.AttrStore return v } @@ -41,7 +41,7 @@ func MustOpenView(db, frame, name string) *View { // Close closes the view and removes all underlying data. func (v *View) Close() error { defer os.Remove(v.Path()) - defer v.BitmapAttrStore.Close() + defer v.RowAttrStore.Close() return v.View.Close() } @@ -53,27 +53,27 @@ func (v *View) Reopen() error { } v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize) - v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore + v.View.RowAttrStore = v.RowAttrStore.AttrStore if err := v.Open(); err != nil { return err } return nil } -// MustSetBits sets bits on a bitmap. Panic on error. +// MustSetBits sets bits on a row. Panic on error. // This function does not accept a timestamp or quantum. -func (v *View) MustSetBits(bitmapID uint64, profileIDs ...uint64) { - for _, profileID := range profileIDs { - if _, err := v.SetBit(bitmapID, profileID); err != nil { +func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := v.SetBit(rowID, columnID); err != nil { panic(err) } } } -// MustClearBits clears bits on a bitmap. Panic on error. -func (v *View) MustClearBits(bitmapID uint64, profileIDs ...uint64) { - for _, profileID := range profileIDs { - if _, err := v.ClearBit(bitmapID, profileID); err != nil { +// MustClearBits clears bits on a row. Panic on error. +func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { + for _, columnID := range columnIDs { + if _, err := v.ClearBit(rowID, columnID); err != nil { panic(err) } }