scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #469 from benbjohnson/nomenclature

Browse source 
Rename bitmap/profile to row/column.
This commit is contained in:
Ben Johnson 2017-04-21 21:25:24 -06:00 • committed by GitHub
commit eec02179c8
33 changed files with 880 additions and 910 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
NOTES
View file

@ -1,10 +1,10 @@
DB Profile
DB Column
┌───────────▼────────────────────────────┐
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
Bitmap──▶0000000000000000000000000000000000000000│
Row──▶0000000000000000000000000000000000000000│
│0000000000000000000000000000000000000000│
│────────────────────────────────────────┤
│0000000000000000000000000000000000000000│

14
README.md
View file

@ -209,17 +209,17 @@ A return value of `{"results":[true]}` indicates that the bit was toggled from 1
A return value of `{"results":[false]}` indicates that the bit was already set to 0 and therefore nothing changed.
---
#### SetBitmapAttrs()
#### SetRowAttrs()
```
SetBitmapAttrs(project=10, frame="collaboration", stars=123, url="http://projects.pilosa.com/10", active=true)
SetRowAttrs(project=10, frame="collaboration", stars=123, url="http://projects.pilosa.com/10", active=true)
```
Returns `{"results":[null]}`
---
#### SetProfileAttrs()
#### SetColumnAttrs()
---
```
SetProfileAttrs(user=10, friends=123, username="mrpi", active=true)
SetColumnAttrs(user=10, friends=123, username="mrpi", active=true)
```
Returns `{"results":[null]}`
@ -230,11 +230,11 @@ Returns `{"results":[null]}`
Bitmap(project=10, frame="collaboration")
```
Returns `{"results":[{"attrs":{"stars":123, "url":"http://projects.pilosa.com/10", "active":true},"bits":[1,2]}]}` where `attrs` are the
attributes set using `SetBitmapAttrs()` and `bits` are the bits set using `SetBit()`.
attributes set using `SetRowAttrs()` and `bits` are the bits set using `SetBit()`.
In order to return profile attributes attached to the profiles of a bitmap, add `&profiles=true` to the query string. Sample response:
In order to return column attributes attached to the columns of a bitmap, add `&columnAttrs=true` to the query string. Sample response:
```
{"results":[{"attrs":{},"bits":[10]}],"profiles":[{"user":10,"attrs":{"friends":123, "username":"mrpi", "active":true}}]}
{"results":[{"attrs":{},"bits":[10]}],"columnAttrs":[{"user":10,"attrs":{"friends":123, "username":"mrpi", "active":true}}]}
```
---

2
attr.go
View file

@ -226,7 +226,7 @@ func (s *AttrStore) BlockData(i uint64) (map[uint64]map[string]interface{}, erro
return m, nil
}
// txAttrs returns a map of attributes for a bitmap.
// txAttrs returns a map of attributes for an id.
func txAttrs(tx *bolt.Tx, id uint64) (map[string]interface{}, error) {
v := tx.Bucket([]byte("attrs")).Get(u64tob(id))
if v == nil {

6
attr_test.go
View file

@ -9,7 +9,7 @@ import (
"github.com/pilosa/pilosa"
)
// Ensure database can set and retrieve profile attributes.
// Ensure database can set and retrieve column attributes.
func TestAttrStore_Attrs(t *testing.T) {
s := MustOpenAttrStore()
defer s.Close()
@ -23,14 +23,14 @@ func TestAttrStore_Attrs(t *testing.T) {
t.Fatal(err)
}
// Retrieve attributes for profile #1.
// Retrieve attributes for column #1.
if m, err := s.Attrs(1); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(100), "B": "VALUE", "C": int64(-27)}) {
t.Fatalf("unexpected attrs(1): %#v", m)
}
// Retrieve attributes for profile #2.
// Retrieve attributes for column #2.
if m, err := s.Attrs(2); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"A": int64(200)}) {

64
cache.go
View file

@ -17,15 +17,15 @@ const (
ThresholdFactor = 1.1
)
// Cache represents a cache for bitmap counts.
// Cache represents a cache of counts.
type Cache interface {
Add(bitmapID uint64, n uint64)
BulkAdd(bitmapID uint64, n uint64)
Get(bitmapID uint64) uint64
Add(id uint64, n uint64)
BulkAdd(id uint64, n uint64)
Get(id uint64) uint64
Len() int
// Returns a list of all bitmap IDs.
BitmapIDs() []uint64
// Returns a list of all IDs.
IDs() []uint64
// Updates the cache, if necessary.
Invalidate()
@ -53,19 +53,19 @@ func NewLRUCache(maxEntries uint32) *LRUCache {
return c
}
func (c *LRUCache) BulkAdd(bitmapID, n uint64) {
c.Add(bitmapID, n)
func (c *LRUCache) BulkAdd(id, n uint64) {
c.Add(id, n)
}
// Add adds a bitmap to the cache.
func (c *LRUCache) Add(bitmapID, n uint64) {
c.cache.Add(bitmapID, n)
c.counts[bitmapID] = n
// Add adds a count to the cache.
func (c *LRUCache) Add(id, n uint64) {
c.cache.Add(id, n)
c.counts[id] = n
}
// Get returns a bitmap with a given id.
func (c *LRUCache) Get(bitmapID uint64) uint64 {
n, _ := c.cache.Get(bitmapID)
// Get returns a count for a given id.
func (c *LRUCache) Get(id uint64) uint64 {
n, _ := c.cache.Get(id)
nn, _ := n.(uint64)
return nn
}
@ -79,8 +79,8 @@ func (c *LRUCache) Invalidate() {}
// Recalculate is a no-op.
func (c *LRUCache) Recalculate() {}
// BitmapIDs returns a list of all bitmap IDs in the cache.
func (c *LRUCache) BitmapIDs() []uint64 {
// IDs returns a list of all IDs in the cache.
func (c *LRUCache) IDs() []uint64 {
a := make([]uint64, 0, len(c.counts))
for id := range c.counts {
a = append(a, id)
@ -136,36 +136,36 @@ func NewRankCache(maxEntries uint32) *RankCache {
}
}
// Add adds a bitmap to the cache.
func (c *RankCache) Add(bitmapID uint64, n uint64) {
// Add adds a count to the cache.
func (c *RankCache) Add(id uint64, n uint64) {
c.mu.Lock()
defer c.mu.Unlock()
// Ignore if the bit count on the bitmap is below the threshold.
// Ignore if the bit count is below the threshold.
if n < c.thresholdValue {
return
}
c.entries[bitmapID] = n
c.entries[id] = n
c.invalidate()
}
// BulkAdd adds a bitmap to the cache unsorted. You should Invalidate after completion.
func (c *RankCache) BulkAdd(bitmapID uint64, n uint64) {
// BulkAdd adds a count to the cache unsorted. You should Invalidate after completion.
func (c *RankCache) BulkAdd(id uint64, n uint64) {
c.mu.Lock()
defer c.mu.Unlock()
if n < c.thresholdValue {
return
}
c.entries[bitmapID] = n
c.entries[id] = n
}
// Get returns a bitmap with a given id.
func (c *RankCache) Get(bitmapID uint64) uint64 {
// Get returns a count for a given id.
func (c *RankCache) Get(id uint64) uint64 {
c.mu.Lock()
defer c.mu.Unlock()
return c.entries[bitmapID]
return c.entries[id]
}
// Len returns the number of items in the cache.
@ -175,8 +175,8 @@ func (c *RankCache) Len() int {
return len(c.entries)
}
// BitmapIDs returns a list of all bitmap IDs in the cache.
func (c *RankCache) BitmapIDs() []uint64 {
// IDs returns a list of all IDs in the cache.
func (c *RankCache) IDs() []uint64 {
c.mu.Lock()
defer c.mu.Unlock()
a := make([]uint64, 0, len(c.entries))
@ -242,7 +242,7 @@ func (c *RankCache) recalculate() {
}
}
// Top returns an ordered list of bitmaps.
// Top returns an ordered list of pairs.
func (c *RankCache) Top() []BitmapPair { return c.rankings }
// WriteTo writes the cache to w.
@ -258,7 +258,7 @@ func (c *RankCache) ReadFrom(r io.Reader) (n int64, err error) {
// Ensure RankCache implements Cache.
var _ Cache = &RankCache{}
// BitmapPair represents a bitmap with an associated identifier.
// BitmapPair represents a id/count pair with an associated identifier.
type BitmapPair struct {
ID uint64
Count uint64
@ -271,7 +271,7 @@ func (p BitmapPairs) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p BitmapPairs) Len() int { return len(p) }
func (p BitmapPairs) Less(i, j int) bool { return p[i].Count > p[j].Count }
// Pair holds a bitmap id and its count.
// Pair holds an id/count pair.
type Pair struct {
ID uint64 `json:"id"`
Count uint64 `json:"count"`

50
client.go
View file

@ -316,9 +316,9 @@ func (c *Client) Import(ctx context.Context, db, frame string, slice uint64, bit
}
func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, error) {
// Separate bitmap and profile IDs to reduce allocations.
bitmapIDs := Bits(bits).BitmapIDs()
profileIDs := Bits(bits).ProfileIDs()
// Separate row and column IDs to reduce allocations.
rowIDs := Bits(bits).RowIDs()
columnIDs := Bits(bits).ColumnIDs()
timestamps := Bits(bits).Timestamps()
// Marshal bits to protobufs.
@ -326,8 +326,8 @@ func MarshalImportPayload(db, frame string, slice uint64, bits []Bit) ([]byte, e
DB: db,
Frame: frame,
Slice: slice,
BitmapIDs: bitmapIDs,
ProfileIDs: profileIDs,
RowIDs: rowIDs,
ColumnIDs: columnIDs,
Timestamps: timestamps,
})
if err != nil {
@ -823,7 +823,7 @@ func (c *Client) FragmentBlocks(ctx context.Context, db, frame, view string, sli
return rsp.Blocks, nil
}
// BlockData returns bitmap/profile id pairs for a block.
// BlockData returns row/column id pairs for a block.
func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice uint64, block int) ([]uint64, []uint64, error) {
buf, err := proto.Marshal(&internal.BlockDataRequest{
DB: db,
@ -867,11 +867,11 @@ func (c *Client) BlockData(ctx context.Context, db, frame, view string, slice ui
} else if err := proto.Unmarshal(body, &rsp); err != nil {
return nil, nil, err
}
return rsp.BitmapIDs, rsp.ProfileIDs, nil
return rsp.RowIDs, rsp.ColumnIDs, nil
}
// ProfileAttrDiff returns data from differing blocks on a remote host.
func (c *Client) ProfileAttrDiff(ctx context.Context, db string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
// ColumnAttrDiff returns data from differing blocks on a remote host.
func (c *Client) ColumnAttrDiff(ctx context.Context, db string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
u := url.URL{
Scheme: "http",
Host: c.host,
@ -913,8 +913,8 @@ func (c *Client) ProfileAttrDiff(ctx context.Context, db string, blks []AttrBloc
return rsp.Attrs, nil
}
// BitmapAttrDiff returns data from differing blocks on a remote host.
func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
// RowAttrDiff returns data from differing blocks on a remote host.
func (c *Client) RowAttrDiff(ctx context.Context, db, frame string, blks []AttrBlock) (map[uint64]map[string]interface{}, error) {
u := url.URL{
Scheme: "http",
Host: c.host,
@ -960,8 +960,8 @@ func (c *Client) BitmapAttrDiff(ctx context.Context, db, frame string, blks []At
// Bit represents the location of a single bit.
type Bit struct {
BitmapID uint64
ProfileID uint64
RowID uint64
ColumnID uint64
Timestamp int64
}
@ -972,29 +972,29 @@ func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p Bits) Len() int { return len(p) }
func (p Bits) Less(i, j int) bool {
if p[i].BitmapID == p[j].BitmapID {
if p[i].ProfileID < p[j].ProfileID {
if p[i].RowID == p[j].RowID {
if p[i].ColumnID < p[j].ColumnID {
return p[i].Timestamp < p[j].Timestamp
}
return p[i].ProfileID < p[j].ProfileID
return p[i].ColumnID < p[j].ColumnID
}
return p[i].BitmapID < p[j].BitmapID
return p[i].RowID < p[j].RowID
}
// BitmapIDs returns a slice of all the bitmap IDs.
func (a Bits) BitmapIDs() []uint64 {
// RowIDs returns a slice of all the row IDs.
func (a Bits) RowIDs() []uint64 {
other := make([]uint64, len(a))
for i := range a {
other[i] = a[i].BitmapID
other[i] = a[i].RowID
}
return other
}
// ProfileIDs returns a slice of all the profile IDs.
func (a Bits) ProfileIDs() []uint64 {
// ColumnIDs returns a slice of all the column IDs.
func (a Bits) ColumnIDs() []uint64 {
other := make([]uint64, len(a))
for i := range a {
other[i] = a[i].ProfileID
other[i] = a[i].ColumnID
}
return other
}
@ -1012,7 +1012,7 @@ func (a Bits) Timestamps() []int64 {
func (a Bits) GroupBySlice() map[uint64][]Bit {
m := make(map[uint64][]Bit)
for _, bit := range a {
slice := bit.ProfileID / SliceWidth
slice := bit.ColumnID / SliceWidth
m[slice] = append(m[slice], bit)
}
@ -1030,7 +1030,7 @@ type BitsByPos []Bit
func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
func (p BitsByPos) Len() int { return len(p) }
func (p BitsByPos) Less(i, j int) bool {
p0, p1 := Pos(p[i].BitmapID, p[i].ProfileID), Pos(p[j].BitmapID, p[j].ProfileID)
p0, p1 := Pos(p[i].RowID, p[i].ColumnID), Pos(p[j].RowID, p[j].ColumnID)
if p0 == p1 {
return p[i].Timestamp < p[j].Timestamp
}

38
client_test.go
View file

@ -28,7 +28,7 @@ func createCluster(c *pilosa.Cluster) ([]*Server, []*Index) {
return server, idx
}
// Test distributed TopN Bitmap count across 3 nodes.
// Test distributed TopN Row count across 3 nodes.
func TestClient_MultiNode(t *testing.T) {
cluster := NewCluster(3)
s, idx := createCluster(cluster)
@ -162,7 +162,7 @@ func TestClient_Import(t *testing.T) {
// Load bitmap into cache to ensure cache gets updated.
f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
f.Bitmap(0)
f.Row(0)
s := NewServer()
defer s.Close()
@ -174,18 +174,18 @@ func TestClient_Import(t *testing.T) {
// Send import request.
c := MustNewClient(s.Host())
if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{
{BitmapID: 0, ProfileID: 1},
{BitmapID: 0, ProfileID: 5},
{BitmapID: 200, ProfileID: 6},
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
{RowID: 200, ColumnID: 6},
}); err != nil {
t.Fatal(err)
}
// Verify data.
if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) {
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) {
t.Fatalf("unexpected bits: %+v", a)
}
if a := f.Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) {
if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) {
t.Fatalf("unexpected bits: %+v", a)
}
}
@ -213,7 +213,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
}
// Load bitmap into cache to ensure cache gets updated.
f.Bitmap(0)
f.Row(0)
s := NewServer()
defer s.Close()
@ -225,22 +225,22 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
// Send import request.
c := MustNewClient(s.Host())
if err := c.Import(context.Background(), "d", "f", 0, []pilosa.Bit{
{BitmapID: 0, ProfileID: 1},
{BitmapID: 0, ProfileID: 5},
{BitmapID: 200, ProfileID: 5},
{BitmapID: 200, ProfileID: 6},
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
{RowID: 200, ColumnID: 5},
{RowID: 200, ColumnID: 6},
}); err != nil {
t.Fatal(err)
}
// Verify data.
if a := f.Bitmap(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) {
if a := f.Row(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected bits: %+v", a)
}
if a := f.Bitmap(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) {
if a := f.Row(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) {
t.Fatalf("unexpected bits: %+v", a)
}
if a := f.Bitmap(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) {
if a := f.Row(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) {
t.Fatalf("unexpected bits: %+v", a)
}
}
@ -279,16 +279,16 @@ func TestClient_BackupRestore(t *testing.T) {
}
// Verify data.
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Bitmap(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) {
if a := idx.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) {
t.Fatalf("unexpected bits: %+v", a)
}
}

2
cmd/export.go
View file

@ -23,7 +23,7 @@ the output is written to STDOUT.
The format of the CSV file is:
BITMAPID,PROFILEID
ROWID,COLUMNID
The file does not contain any headers.
`,

2
cmd/import.go
View file

@ -21,7 +21,7 @@ of the CSV file are grouped by slice for the most efficient import.
The format of the CSV file is:
BITMAPID,PROFILEID,[TIME]
ROWID,COLUMNID,[TIME]
The file should contain no headers. The TIME column is optional and can be
omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.

2
cmd/sort.go
View file

@ -25,7 +25,7 @@ Sorts the import data at PATH into the optimal sort order for importing.
The format of the CSV file is:
BITMAPID,PROFILEID
ROWID,COLUMNID
The file should contain no headers.
`,

10
ctl/bench.go
View file

@ -63,17 +63,17 @@ func (cmd *BenchCommand) runSetBit(ctx context.Context, client *pilosa.Client) e
return pilosa.ErrFrameRequired
}
const maxBitmapID = 1000
const maxProfileID = 100000
const maxRowID = 1000
const maxColumnID = 100000
startTime := time.Now()
// Execute operation continuously.
for i := 0; i < cmd.N; i++ {
bitmapID := rand.Intn(maxBitmapID)
profileID := rand.Intn(maxProfileID)
rowID := rand.Intn(maxRowID)
columnID := rand.Intn(maxColumnID)
q := fmt.Sprintf(`SetBit(id=%d, frame="%s", profileID=%d)`, bitmapID, cmd.Frame, profileID)
q := fmt.Sprintf(`SetBit(id=%d, frame="%s", columnID=%d)`, rowID, cmd.Frame, columnID)
if _, err := client.ExecuteQuery(ctx, cmd.Database, q, true); err != nil {
return err

16
ctl/import.go
View file

@ -123,19 +123,19 @@ func (cmd *ImportCommand) importPath(ctx context.Context, path string) error {
var bit pilosa.Bit
// Parse bitmap id.
bitmapID, err := strconv.ParseUint(record[0], 10, 64)
// Parse row id.
rowID, err := strconv.ParseUint(record[0], 10, 64)
if err != nil {
return fmt.Errorf("invalid bitmap id on row %d: %q", rnum, record[0])
return fmt.Errorf("invalid row id on row %d: %q", rnum, record[0])
}
bit.BitmapID = bitmapID
bit.RowID = rowID
// Parse bitmap id.
profileID, err := strconv.ParseUint(record[1], 10, 64)
// Parse column id.
columnID, err := strconv.ParseUint(record[1], 10, 64)
if err != nil {
return fmt.Errorf("invalid profile id on row %d: %q", rnum, record[1])
return fmt.Errorf("invalid column id on row %d: %q", rnum, record[1])
}
bit.ProfileID = profileID
bit.ColumnID = columnID
// Parse time, if exists.
if len(record) > 2 && record[2] != "" {

26
ctl/sort.go
View file

@ -45,7 +45,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error {
r.FieldsPerRecord = -1
a := make([]pilosa.Bit, 0, 1000000)
for {
bitmapID, profileID, timestamp, err := readCSVRow(r)
rowID, columnID, timestamp, err := readCSVRow(r)
if err == io.EOF {
break
} else if err == errBlank {
@ -53,7 +53,7 @@ func (cmd *SortCommand) Run(ctx context.Context) error {
} else if err != nil {
return err
}
a = append(a, pilosa.Bit{BitmapID: bitmapID, ProfileID: profileID, Timestamp: timestamp})
a = append(a, pilosa.Bit{RowID: rowID, ColumnID: columnID, Timestamp: timestamp})
}
// Sort bits by position.
@ -65,10 +65,10 @@ func (cmd *SortCommand) Run(ctx context.Context) error {
for _, bit := range a {
// Write CSV to buffer.
buf = buf[:0]
buf = strconv.AppendUint(buf, bit.BitmapID, 10)
buf = strconv.AppendUint(buf, bit.RowID, 10)
buf = append(buf, ',')
buf = strconv.AppendUint(buf, bit.ProfileID, 10)
buf = strconv.AppendUint(buf, bit.ColumnID, 10)
if bit.Timestamp != 0 {
buf = append(buf, ',')
@ -91,8 +91,8 @@ func (cmd *SortCommand) Run(ctx context.Context) error {
return nil
}
// readCSVRow reads a bitmap/profile pair from a CSV row.
func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err error) {
// readCSVRow reads a row/column pair from a CSV row.
func readCSVRow(r *csv.Reader) (rowID, columnID uint64, timestamp int64, err error) {
// Read CSV row.
record, err := r.Read()
if err != nil {
@ -106,16 +106,16 @@ func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err
return 0, 0, 0, fmt.Errorf("bad column count: %d", len(record))
}
// Parse bitmap id.
bitmapID, err = strconv.ParseUint(record[0], 10, 64)
// Parse row id.
rowID, err = strconv.ParseUint(record[0], 10, 64)
if err != nil {
return 0, 0, 0, fmt.Errorf("invalid bitmap id: %q", record[0])
return 0, 0, 0, fmt.Errorf("invalid row id: %q", record[0])
}
// Parse bitmap id.
profileID, err = strconv.ParseUint(record[1], 10, 64)
// Parse column id.
columnID, err = strconv.ParseUint(record[1], 10, 64)
if err != nil {
return 0, 0, 0, fmt.Errorf("invalid profile id: %q", record[1])
return 0, 0, 0, fmt.Errorf("invalid column id: %q", record[1])
}
// Parse timestamp, if available.
@ -127,7 +127,7 @@ func readCSVRow(r *csv.Reader) (bitmapID, profileID uint64, timestamp int64, err
timestamp = t.UnixNano()
}
return bitmapID, profileID, timestamp, nil
return rowID, columnID, timestamp, nil
}
// errBlank indicates a blank row in a CSV file.

22
db.go
View file

@ -17,7 +17,7 @@ import (
// Default database settings.
const (
DefaultColumnLabel = "profileID"
DefaultColumnLabel = "columnID"
)
// DB represents a container for frames.
@ -40,8 +40,8 @@ type DB struct {
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
// Profile attribute storage and cache
profileAttrStore *AttrStore
// Column attribute storage and cache
columnAttrStore *AttrStore
broadcaster Broadcaster
stats StatsClient
@ -64,7 +64,7 @@ func NewDB(path, name string) (*DB, error) {
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
profileAttrStore: NewAttrStore(filepath.Join(path, ".data")),
columnAttrStore: NewAttrStore(filepath.Join(path, ".data")),
columnLabel: DefaultColumnLabel,
@ -79,8 +79,8 @@ func (db *DB) Name() string { return db.name }
// Path returns the path the database was initialized with.
func (db *DB) Path() string { return db.path }
// ProfileAttrStore returns the storage for profile attributes.
func (db *DB) ProfileAttrStore() *AttrStore { return db.profileAttrStore }
// ColumnAttrStore returns the storage for column attributes.
func (db *DB) ColumnAttrStore() *AttrStore { return db.columnAttrStore }
// SetColumnLabel sets the column label. Persists to meta file on update.
func (db *DB) SetColumnLabel(v string) error {
@ -131,7 +131,7 @@ func (db *DB) Open() error {
return err
}
if err := db.profileAttrStore.Open(); err != nil {
if err := db.columnAttrStore.Open(); err != nil {
return err
}
@ -220,8 +220,8 @@ func (db *DB) Close() error {
defer db.mu.Unlock()
// Close the attribute store.
if db.profileAttrStore != nil {
db.profileAttrStore.Close()
if db.columnAttrStore != nil {
db.columnAttrStore.Close()
}
// Close all frames.
@ -560,6 +560,6 @@ type importKey struct {
}
type importData struct {
BitmapIDs []uint64
ProfileIDs []uint64
RowIDs []uint64
ColumnIDs []uint64
}

76
executor.go
View file

@ -21,7 +21,7 @@ const (
DefaultFrame = "general"
// MinThreshold is the lowest count to use in a Top-N operation when
// looking for additional bitmap/count pairs.
// looking for additional id/count pairs.
MinThreshold = 1
)
@ -71,8 +71,8 @@ func (e *Executor) Execute(ctx context.Context, db string, q *pql.Query, slices
}
// Optimize handling for bulk attribute insertion.
if hasOnlySetBitmapAttrs(q.Calls) {
return e.executeBulkSetBitmapAttrs(ctx, db, q.Calls, opt)
if hasOnlySetRowAttrs(q.Calls) {
return e.executeBulkSetRowAttrs(ctx, db, q.Calls, opt)
}
// Execute each call serially.
@ -102,10 +102,10 @@ func (e *Executor) executeCall(ctx context.Context, db string, c *pql.Call, slic
return e.executeCount(ctx, db, c, slices, opt)
case "SetBit":
return e.executeSetBit(ctx, db, c, opt)
case "SetBitmapAttrs":
return nil, e.executeSetBitmapAttrs(ctx, db, c, opt)
case "SetProfileAttrs":
return nil, e.executeSetProfileAttrs(ctx, db, c, opt)
case "SetRowAttrs":
return nil, e.executeSetRowAttrs(ctx, db, c, opt)
case "SetColumnAttrs":
return nil, e.executeSetColumnAttrs(ctx, db, c, opt)
case "TopN":
return e.executeTopN(ctx, db, c, slices, opt)
default:
@ -155,7 +155,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
}
// Attach attributes for Bitmap() calls.
// If the column label is used then return profile attributes.
// If the column label is used then return column attributes.
// If the row label is used then return bitmap attributes.
bm, _ := other.(*Bitmap)
if c.Name == "Bitmap" {
@ -164,7 +164,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
if d != nil {
columnLabel := d.ColumnLabel()
if columnID, ok, err := c.UintArg(columnLabel); ok && err == nil {
attrs, err := d.ProfileAttrStore().Attrs(columnID)
attrs, err := d.ColumnAttrStore().Attrs(columnID)
if err != nil {
return nil, err
}
@ -179,7 +179,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, db string, c *pql.Call
if err != nil {
return nil, err
}
attrs, err := fr.BitmapAttrStore().Attrs(rowID)
attrs, err := fr.RowAttrStore().Attrs(rowID)
if err != nil {
return nil, err
}
@ -214,7 +214,7 @@ func (e *Executor) executeBitmapCallSlice(ctx context.Context, db string, c *pql
// This first performs the TopN() to determine the top results and then
// requeries to retrieve the full counts for each of the top results.
func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slices []uint64, opt *ExecOptions) ([]Pair, error) {
bitmapIDs, _, err := c.UintSliceArg("ids")
rowIDs, _, err := c.UintSliceArg("ids")
if err != nil {
return nil, fmt.Errorf("executeTopN: %v", err)
}
@ -231,7 +231,7 @@ func (e *Executor) executeTopN(ctx context.Context, db string, c *pql.Call, slic
// If this call is against specific ids, or we didn't get results,
// or we are part of a larger distributed query then don't refetch.
if len(pairs) == 0 || len(bitmapIDs) > 0 || opt.Remote {
if len(pairs) == 0 || len(rowIDs) > 0 || opt.Remote {
return pairs, nil
}
// Only the original caller should refetch the full counts.
@ -284,7 +284,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
return nil, fmt.Errorf("executeTopNSlice: %v", err)
}
field, _ := c.Args["field"].(string)
bitmapIDs, _, err := c.UintSliceArg("ids")
rowIDs, _, err := c.UintSliceArg("ids")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
}
@ -330,7 +330,7 @@ func (e *Executor) executeTopNSlice(ctx context.Context, db string, c *pql.Call,
return f.Top(TopOptions{
N: int(n),
Src: src,
BitmapIDs: bitmapIDs,
RowIDs: rowIDs,
FilterField: field,
FilterValues: filters,
MinThreshold: minThreshold,
@ -404,7 +404,7 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, db string, c *pql.Cal
if frag == nil {
return NewBitmap(), nil
}
return frag.Bitmap(id), nil
return frag.Row(id), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
@ -483,7 +483,7 @@ func (e *Executor) executeRangeSlice(ctx context.Context, db string, c *pql.Call
if f == nil {
continue
}
bm = bm.Union(f.Bitmap(rowID))
bm = bm.Union(f.Row(rowID))
}
return bm, nil
}
@ -739,11 +739,11 @@ func (e *Executor) executeSetBitView(ctx context.Context, db string, c *pql.Call
return ret, nil
}
// executeSetBitmapAttrs executes a SetBitmapAttrs() call.
func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
// executeSetRowAttrs executes a SetRowAttrs() call.
func (e *Executor) executeSetRowAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
frameName, ok := c.Args["frame"].(string)
if !ok {
return errors.New("SetBitmapAttrs() frame required")
return errors.New("SetRowAttrs() frame required")
}
// Retrieve frame.
@ -756,9 +756,9 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
// Parse labels.
rowID, ok, err := c.UintArg(rowLabel)
if err != nil {
return fmt.Errorf("reading SetBitmapAttrs() row: %v", err)
return fmt.Errorf("reading SetRowAttrs() row: %v", err)
} else if !ok {
return fmt.Errorf("SetBitmapAttrs() row field '%v' required.", rowLabel)
return fmt.Errorf("SetRowAttrs() row field '%v' required.", rowLabel)
}
// Copy args and remove reserved fields.
@ -767,7 +767,7 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
delete(attrs, rowLabel)
// Set attributes.
if err := frame.BitmapAttrStore().SetAttrs(rowID, attrs); err != nil {
if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
return err
}
@ -796,14 +796,14 @@ func (e *Executor) executeSetBitmapAttrs(ctx context.Context, db string, c *pql.
return nil
}
// executeBulkSetBitmapAttrs executes a set of SetBitmapAttrs() calls.
func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) {
// executeBulkSetRowAttrs executes a set of SetRowAttrs() calls.
func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, db string, calls []*pql.Call, opt *ExecOptions) ([]interface{}, error) {
// Collect attributes by frame/id.
m := make(map[string]map[uint64]map[string]interface{})
for _, c := range calls {
frame, ok := c.Args["frame"].(string)
if !ok {
return nil, errors.New("SetBitmapAttrs() frame required")
return nil, errors.New("SetRowAttrs() frame required")
}
// Retrieve frame.
@ -815,9 +815,9 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
rowID, ok, err := c.UintArg(rowLabel)
if err != nil {
return nil, fmt.Errorf("reading SetBitmapAttrs() row: %v", rowLabel)
return nil, fmt.Errorf("reading SetRowAttrs() row: %v", rowLabel)
} else if !ok {
return nil, fmt.Errorf("SetBitmapAttrs row field '%v' required.", rowLabel)
return nil, fmt.Errorf("SetRowAttrs row field '%v' required.", rowLabel)
}
// Copy args and remove reserved fields.
@ -852,7 +852,7 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
}
// Set attributes.
if err := frame.BitmapAttrStore().SetBulkAttrs(frameMap); err != nil {
if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil {
return nil, err
}
}
@ -883,8 +883,8 @@ func (e *Executor) executeBulkSetBitmapAttrs(ctx context.Context, db string, cal
return make([]interface{}, len(calls)), nil
}
// executeSetProfileAttrs executes a SetProfileAttrs() call.
func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
// executeSetColumnAttrs executes a SetColumnAttrs() call.
func (e *Executor) executeSetColumnAttrs(ctx context.Context, db string, c *pql.Call, opt *ExecOptions) error {
// Retrieve database.
d := e.Index.DB(db)
if d == nil {
@ -898,7 +898,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql
columnLabel := d.columnLabel
col, okCol, errCol := c.UintArg(columnLabel)
if errCol != nil || !okCol {
return fmt.Errorf("reading SetProfileAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol)
return fmt.Errorf("reading SetColumnAttrs() id/columnLabel errs: %v/%v found %v/%v", errID, errCol, okID, okCol)
}
id = col
colName = columnLabel
@ -911,7 +911,7 @@ func (e *Executor) executeSetProfileAttrs(ctx context.Context, db string, c *pql
delete(attrs, colName)
// Set attributes.
if err := d.ProfileAttrStore().SetAttrs(id, attrs); err != nil {
if err := d.ColumnAttrStore().SetAttrs(id, attrs); err != nil {
return err
}
@ -1011,8 +1011,8 @@ func (e *Executor) exec(ctx context.Context, node *Node, db string, q *pql.Query
v, err = pb.Results[i].Changed, nil
case "ClearBit":
v, err = pb.Results[i].Changed, nil
case "SetBitmapAttrs":
case "SetProfileAttrs":
case "SetRowAttrs":
case "SetColumnAttrs":
default:
v, err = decodeBitmap(pb.Results[i].GetBitmap()), nil
}
@ -1207,14 +1207,14 @@ func decodeError(s string) error {
return errors.New(s)
}
// hasOnlySetBitmapAttrs returns true if calls only contains SetBitmapAttrs() calls.
func hasOnlySetBitmapAttrs(calls []*pql.Call) bool {
// hasOnlySetRowAttrs returns true if calls only contains SetRowAttrs() calls.
func hasOnlySetRowAttrs(calls []*pql.Call) bool {
if len(calls) == 0 {
return false
}
for _, call := range calls {
if call.Name != "SetBitmapAttrs" {
if call.Name != "SetRowAttrs" {
return false
}
}
@ -1227,7 +1227,7 @@ func needsSlices(calls []*pql.Call) bool {
}
for _, call := range calls {
switch call.Name {
case "ClearBit", "SetBit", "SetBitmapAttrs", "SetProfileAttrs":
case "ClearBit", "SetBit", "SetRowAttrs", "SetColumnAttrs":
continue
case "Count", "TopN":
return true

66
executor_test.go
View file

@ -27,13 +27,13 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "d", MustParse(``+
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1),
), nil, nil); err != nil {
t.Fatal(err)
}
if err := f.BitmapAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
if err := f.RowAttrStore().SetAttrs(10, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
t.Fatal(err)
}
@ -58,17 +58,17 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Set bits.
if _, err := e.Execute(context.Background(), "d", MustParse(``+
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, id=%d, profileID=%d)\n", 20, SliceWidth+1),
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, id=%d, columnID=%d)\n", 20, SliceWidth+1),
), nil, nil); err != nil {
t.Fatal(err)
}
if err := db.ProfileAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
if err := db.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "d", MustParse(fmt.Sprintf(`Bitmap(profileID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "d", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -195,11 +195,11 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
e := NewExecutor(idx.Index, NewCluster(1))
f := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0)
if n := f.Bitmap(11).Count(); n != 0 {
if n := f.Row(11).Count(); n != 0 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if !res[0].(bool) {
@ -207,10 +207,10 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
}
}
if n := f.Bitmap(11).Count(); n != 1 {
if n := f.Row(11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, profileID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=11, frame=f, columnID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if res[0].(bool) {
@ -219,8 +219,8 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
}
}
// Ensure a SetBitmapAttrs() query can be executed.
func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
// Ensure a SetRowAttrs() query can be executed.
func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
@ -235,21 +235,21 @@ func TestExecutor_Execute_SetBitmapAttrs(t *testing.T) {
// Set two fields on f/10.
// Also set fields on other bitmaps and frames to test isolation.
e := NewExecutor(idx.Index, NewCluster(1))
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=200, frame=f, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBitmapAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(id=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
}
f := idx.Frame("d", "f")
if m, err := f.BitmapAttrStore().Attrs(10); err != nil {
if m, err := f.RowAttrStore().Attrs(10); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(m, map[string]interface{}{"foo": "bar", "baz": int64(123), "bat": true}) {
t.Fatalf("unexpected bitmap attr: %#v", m)
@ -261,7 +261,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
// Set bits for bitmaps 0, 10, & 20 across two slices.
// Set bits for rows 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
@ -287,7 +287,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
// Set bits for bitmaps 0, 10, & 20 across two slices.
// Set bits for rows 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
@ -345,7 +345,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
idx := MustOpenIndex()
defer idx.Close()
// Set bits for bitmaps 0, 10, & 20 across two slices.
// Set bits for rows 0, 10, & 20 across two slices.
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
@ -355,7 +355,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(20, SliceWidth+2)
// Create an intersecting bitmap.
// Create an intersecting row.
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth)
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+1)
idx.MustCreateFragmentIfNotExists("d", "other", pilosa.ViewStandard, 1).SetBit(100, SliceWidth+2)
@ -382,7 +382,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
if err := idx.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
}
e := NewExecutor(idx.Index, NewCluster(1))
@ -405,7 +405,7 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 1).SetBit(10, SliceWidth)
if err := idx.Frame("d", "f").BitmapAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
if err := idx.Frame("d", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
}
e := NewExecutor(idx.Index, NewCluster(1))
@ -445,7 +445,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
f.MustSetBit(pilosa.ViewStandard, 1, 2, MustParseTimePtr("1999-12-30 00:00")) // too early
f.MustSetBit(pilosa.ViewStandard, 1, 2, MustParseTimePtr("2002-02-01 00:00")) // too late
f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different bitmap
f.MustSetBit(pilosa.ViewStandard, 10, 2, MustParseTimePtr("2001-01-01 00:00")) // different row
e := NewExecutor(idx.Index, NewCluster(1))
if res, err := e.Execute(context.Background(), "d", MustParse(`Range(id=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
@ -542,7 +542,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `SetBit(frame="f", id=10, profileID=2)` {
} else if query.String() != `SetBit(columnID=2, frame="f", id=10)` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -559,12 +559,12 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
e := NewExecutor(idx.Index, c)
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2)`), nil, nil); err != nil {
t.Fatal(err)
}
// Verify that one bit is set on both node's index.
if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Bitmap(10).Count(); n != 1 {
if n := idx.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
if !remoteCalled {
@ -587,7 +587,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
s.Handler.Executor.ExecuteFn = func(ctx context.Context, db string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if db != `d` {
t.Fatalf("unexpected db: %s", db)
} else if query.String() != `SetBit(frame="f", id=10, profileID=2, timestamp="2016-12-11T10:09")` {
} else if query.String() != `SetBit(columnID=2, frame="f", id=10, timestamp="2016-12-11T10:09")` {
t.Fatalf("unexpected query: %s", query.String())
}
remoteCalled = true
@ -606,12 +606,12 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
}
e := NewExecutor(idx.Index, c)
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, profileID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", MustParse(`SetBit(id=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
t.Fatal(err)
}
// Verify that one bit is set on both node's index.
if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Bitmap(10).Count(); n != 1 {
if n := idx.MustCreateFragmentIfNotExists("d", "f", "standard_2016", 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
if !remoteCalled {

292
fragment.go
View file

@ -29,7 +29,7 @@ import (
)
const (
// SliceWidth is the number of profile IDs in a slice.
// SliceWidth is the number of column IDs in a slice.
SliceWidth = 1048576
// SnapshotExt is the file extension used for an in-process snapshot.
@ -41,7 +41,7 @@ const (
// CacheExt is the file extension for persisted cache ids.
CacheExt = ".cache"
// HashBlockSize is the number of bitmaps in a merkle hash block.
// HashBlockSize is the number of rows in a merkle hash block.
HashBlockSize = 100
)
@ -67,13 +67,13 @@ type Fragment struct {
storageData []byte
opN int // number of ops since snapshot
// Cache for bitmap counts.
// Cache for row counts.
cacheType string // passed in by frame
cache Cache
cacheSize uint32
// Cache containing full bitmaps (not just counts).
bitmapCache BitmapCache
// Cache containing full rows (not just counts).
rowCache BitmapCache
// Cached checksums for each block.
checksums map[int][]byte
@ -86,9 +86,9 @@ type Fragment struct {
// Writer used for out-of-band log entries.
LogOutput io.Writer
// Bitmap attribute storage.
// Row attribute storage.
// This is set by the parent frame unless overridden for testing.
BitmapAttrStore *AttrStore
RowAttrStore *AttrStore
stats StatsClient
}
@ -144,7 +144,7 @@ func (f *Fragment) Open() error {
return err
}
// Fill cache with bitmaps persisted to disk.
// Fill cache with rows persisted to disk.
if err := f.openCache(); err != nil {
return err
}
@ -213,13 +213,13 @@ func (f *Fragment) openStorage() error {
// Attach the file to the bitmap to act as a write-ahead log.
f.storage.OpWriter = f.file
f.bitmapCache = &SimpleCache{make(map[uint64]*Bitmap)}
f.rowCache = &SimpleCache{make(map[uint64]*Bitmap)}
return nil
}
// openCache initializes the cache from bitmap ids persisted to disk.
// openCache initializes the cache from row ids persisted to disk.
func (f *Fragment) openCache() error {
// Determine cache type from frame name.
switch f.cacheType {
@ -247,12 +247,12 @@ func (f *Fragment) openCache() error {
return nil
}
// Read in all bitmaps by ID.
// Read in all rows by ID.
// This will cause them to be added to the cache.
for _, bitmapID := range pb.BitmapIDs {
//n := f.storage.CountRange(bitmapID*SliceWidth, (bitmapID+1)*SliceWidth)
n := f.bitmap(bitmapID, true, true).Count()
f.cache.BulkAdd(bitmapID, n)
for _, id := range pb.IDs {
//n := f.storage.CountRange(id*SliceWidth, (id+1)*SliceWidth)
n := f.row(id, true, true).Count()
f.cache.BulkAdd(id, n)
}
f.cache.Invalidate()
@ -314,17 +314,16 @@ func (f *Fragment) closeStorage() error {
// logger returns a logger instance for the fragment.nt.
func (f *Fragment) logger() *log.Logger { return log.New(f.LogOutput, "", log.LstdFlags) }
// Bitmap returns a bitmap by ID.
func (f *Fragment) Bitmap(bitmapID uint64) *Bitmap {
// Row returns a row by ID.
func (f *Fragment) Row(rowID uint64) *Bitmap {
f.mu.Lock()
defer f.mu.Unlock()
return f.bitmap(bitmapID, true, true)
return f.row(rowID, true, true)
}
func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCache bool) *Bitmap {
if checkBitmapCache {
r, ok := f.bitmapCache.Fetch(bitmapID)
func (f *Fragment) row(rowID uint64, checkRowCache bool, updateRowCache bool) *Bitmap {
if checkRowCache {
r, ok := f.rowCache.Fetch(rowID)
if ok && r != nil {
return r
}
@ -332,11 +331,11 @@ func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCa
// Only use a subset of the containers.
// NOTE: The start & end ranges must be divisible by
data := f.storage.OffsetRange(f.slice*SliceWidth, bitmapID*SliceWidth, (bitmapID+1)*SliceWidth)
data := f.storage.OffsetRange(f.slice*SliceWidth, rowID*SliceWidth, (rowID+1)*SliceWidth)
// Reference bitmap subrange in storage.
// We Clone() data because otherwise bm will contains pointers to containers in storage.
// This causes unexpected results when we cache the bitmap and try to use it later.
// This causes unexpected results when we cache the row and try to use it later.
bm := &Bitmap{
segments: []BitmapSegment{{
data: *data.Clone(),
@ -346,25 +345,25 @@ func (f *Fragment) bitmap(bitmapID uint64, checkBitmapCache bool, updateBitmapCa
}
bm.InvalidateCount()
if updateBitmapCache {
f.bitmapCache.Add(bitmapID, bm)
if updateRowCache {
f.rowCache.Add(rowID, bm)
}
return bm
}
// SetBit sets a bit for a given profile & bitmap within the fragment.
// SetBit sets a bit for a given column & row within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) SetBit(bitmapID, profileID uint64) (changed bool, err error) {
func (f *Fragment) SetBit(rowID, columnID uint64) (changed bool, err error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.setBit(bitmapID, profileID)
return f.setBit(rowID, columnID)
}
func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error) {
func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
changed = false
// Determine the position of the bit in the storage.
pos, err := f.pos(bitmapID, profileID)
pos, err := f.pos(rowID, columnID)
if err != nil {
return false, err
}
@ -380,37 +379,37 @@ func (f *Fragment) setBit(bitmapID, profileID uint64) (changed bool, err error)
}
// Invalidate block checksum.
delete(f.checksums, int(bitmapID/HashBlockSize))
delete(f.checksums, int(rowID/HashBlockSize))
// Increment number of operations until snapshot is required.
if err := f.incrementOpN(); err != nil {
return false, err
}
// Get the bitmap from bitmapCache or fragment.storage.
bm := f.bitmap(bitmapID, true, true)
bm.SetBit(profileID)
// Get the row from row cache or fragment.storage.
bm := f.row(rowID, true, true)
bm.SetBit(columnID)
// Update the cache.
f.cache.Add(bitmapID, bm.Count())
f.cache.Add(rowID, bm.Count())
f.stats.Count("setN", 1)
return changed, nil
}
// ClearBit clears a bit for a given profile & bitmap within the fragment.
// ClearBit clears a bit for a given column & row within the fragment.
// This updates both the on-disk storage and the in-cache bitmap.
func (f *Fragment) ClearBit(bitmapID, profileID uint64) (bool, error) {
func (f *Fragment) ClearBit(rowID, columnID uint64) (bool, error) {
f.mu.Lock()
defer f.mu.Unlock()
return f.clearBit(bitmapID, profileID)
return f.clearBit(rowID, columnID)
}
func (f *Fragment) clearBit(bitmapID, profileID uint64) (changed bool, err error) {
func (f *Fragment) clearBit(rowID, columnID uint64) (changed bool, err error) {
changed = false
// Determine the position of the bit in the storage.
pos, err := f.pos(bitmapID, profileID)
pos, err := f.pos(rowID, columnID)
if err != nil {
return false, err
}
@ -426,38 +425,38 @@ func (f *Fragment) clearBit(bitmapID, profileID uint64) (changed bool, err error
}
// Invalidate block checksum.
delete(f.checksums, int(bitmapID/HashBlockSize))
delete(f.checksums, int(rowID/HashBlockSize))
// Increment number of operations until snapshot is required.
if err := f.incrementOpN(); err != nil {
return false, err
}
// Get the bitmap from bitmapCache or fragment.storage.
bm := f.bitmap(bitmapID, true, true)
bm.ClearBit(profileID)
// Get the row from cache or fragment.storage.
bm := f.row(rowID, true, true)
bm.ClearBit(columnID)
// Update the cache.
f.cache.Add(bitmapID, bm.Count())
f.cache.Add(rowID, bm.Count())
f.stats.Count("clearN", 1)
return changed, nil
}
// pos translates the bitmap ID and profile ID into a position in the storage bitmap.
func (f *Fragment) pos(bitmapID, profileID uint64) (uint64, error) {
// Return an error if the profile ID is out of the range of the fragment's slice.
minProfileID := f.slice * SliceWidth
if profileID < minProfileID || profileID >= minProfileID+SliceWidth {
return 0, errors.New("profile out of bounds")
// pos translates the row ID and column ID into a position in the storage bitmap.
func (f *Fragment) pos(rowID, columnID uint64) (uint64, error) {
// Return an error if the column ID is out of the range of the fragment's slice.
minColumnID := f.slice * SliceWidth
if columnID < minColumnID || columnID >= minColumnID+SliceWidth {
return 0, errors.New("column out of bounds")
}
return Pos(bitmapID, profileID), nil
return Pos(rowID, columnID), nil
}
// ForEachBit executes fn for every bit set in the fragment.
// Errors returned from fn are passed through.
func (f *Fragment) ForEachBit(fn func(bitmapID, profileID uint64) error) error {
func (f *Fragment) ForEachBit(fn func(rowID, columnID uint64) error) error {
f.mu.Lock()
defer f.mu.Unlock()
@ -474,15 +473,15 @@ func (f *Fragment) ForEachBit(fn func(bitmapID, profileID uint64) error) error {
return err
}
// Top returns the top bitmaps from the fragment.
// If opt.Src is specified then only bitmaps which intersect src are returned.
// If opt.FilterValues exist then the bitmap attribute specified by field is matched.
// Top returns the top rows from the fragment.
// If opt.Src is specified then only rows which intersect src are returned.
// If opt.FilterValues exist then the row attribute specified by field is matched.
func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Retrieve pairs. If no bitmap ids specified then return from cache.
pairs := f.topBitmapPairs(opt.BitmapIDs)
// Retrieve pairs. If no row ids specified then return from cache.
pairs := f.topBitmapPairs(opt.RowIDs)
// If BitmapIDs are provided, we don't want to truncate the result set
if len(opt.BitmapIDs) > 0 {
// If row ids are provided, we don't want to truncate the result set
if len(opt.RowIDs) > 0 {
opt.N = 0
}
@ -509,9 +508,9 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Iterate over rankings and add to results until we have enough.
results := &PairHeap{}
for _, pair := range pairs {
bitmapID, cnt := pair.ID, pair.Count
rowID, cnt := pair.ID, pair.Count
// Ignore empty bitmaps.
// Ignore empty rows.
if cnt <= 0 {
continue
}
@ -531,7 +530,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Apply filter, if set.
if filters != nil {
attr, err := f.BitmapAttrStore.Attrs(bitmapID)
attr, err := f.RowAttrStore.Attrs(rowID)
if err != nil {
return nil, err
} else if attr == nil {
@ -548,7 +547,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// Calculate count and append.
count := cnt
if opt.Src != nil {
count = opt.Src.IntersectionCount(f.Bitmap(bitmapID))
count = opt.Src.IntersectionCount(f.Row(rowID))
}
if count == 0 {
continue
@ -566,7 +565,7 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
}
}
heap.Push(results, Pair{ID: bitmapID, Count: count})
heap.Push(results, Pair{ID: rowID, Count: count})
// If we reach the requested number of pairs and we are not computing
// intersections then simply exit. If we are intersecting then sort
@ -584,20 +583,20 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// If it's too low then don't try finding anymore pairs.
threshold := results.Pairs[0].Count
// If the bitmap doesn't have enough bits set before the intersection
// then we can assume that any remaining bitmaps also have a count too low.
// If the row doesn't have enough bits set before the intersection
// then we can assume that any remaining rows also have a count too low.
if threshold < opt.MinThreshold || cnt < threshold {
break
}
// Calculate the intersecting bit count and skip if it's below our
// last bitmap in our current result set.
count := opt.Src.IntersectionCount(f.Bitmap(bitmapID))
// last row in our current result set.
count := opt.Src.IntersectionCount(f.Row(rowID))
if count < threshold {
continue
}
heap.Push(results, Pair{ID: bitmapID, Count: count})
heap.Push(results, Pair{ID: rowID, Count: count})
}
//Pop first opt.N elements out of heap
@ -611,32 +610,32 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
return r, nil
}
func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair {
// If no specific bitmaps are requested, retrieve top bitmaps.
if len(bitmapIDs) == 0 {
func (f *Fragment) topBitmapPairs(rowIDs []uint64) []BitmapPair {
// If no specific rows are requested, retrieve top rows.
if len(rowIDs) == 0 {
f.mu.Lock()
defer f.mu.Unlock()
f.cache.Invalidate()
return f.cache.Top()
}
// Otherwise retrieve specific bitmaps.
pairs := make([]BitmapPair, 0, len(bitmapIDs))
for _, bitmapID := range bitmapIDs {
// Otherwise retrieve specific rows.
pairs := make([]BitmapPair, 0, len(rowIDs))
for _, rowID := range rowIDs {
// Look up cache first, if available.
if n := f.cache.Get(bitmapID); n > 0 {
if n := f.cache.Get(rowID); n > 0 {
pairs = append(pairs, BitmapPair{
ID: bitmapID,
ID: rowID,
Count: n,
})
continue
}
bm := f.Bitmap(bitmapID)
bm := f.Row(rowID)
if bm.Count() > 0 {
// Otherwise load from storage.
pairs = append(pairs, BitmapPair{
ID: bitmapID,
ID: rowID,
Count: bm.Count(),
})
}
@ -647,14 +646,14 @@ func (f *Fragment) topBitmapPairs(bitmapIDs []uint64) []BitmapPair {
// TopOptions represents options passed into the Top() function.
type TopOptions struct {
// Number of bitmaps to return.
// Number of rows to return.
N int
// Bitmap to intersect with.
Src *Bitmap
// Specific bitmaps to filter against.
BitmapIDs []uint64
// Specific rows to filter against.
RowIDs []uint64
MinThreshold uint64
// Filter field name & values.
@ -768,14 +767,14 @@ func (f *Fragment) readContiguousChecksums(a *[]FragmentBlock, blockID int) (n i
}
}
// BlockData returns bits in a block as bitmap & profile ID pairs.
func (f *Fragment) BlockData(id int) (bitmapIDs, profileIDs []uint64) {
// BlockData returns bits in a block as row & column ID pairs.
func (f *Fragment) BlockData(id int) (rowIDs, columnIDs []uint64) {
f.mu.Lock()
defer f.mu.Unlock()
f.storage.ForEachRange(uint64(id)*HashBlockSize*SliceWidth, (uint64(id)+1)*HashBlockSize*SliceWidth, func(i uint64) {
bitmapIDs = append(bitmapIDs, i/SliceWidth)
profileIDs = append(profileIDs, i%SliceWidth)
rowIDs = append(rowIDs, i/SliceWidth)
columnIDs = append(columnIDs, i%SliceWidth)
})
return
}
@ -789,8 +788,8 @@ func (f *Fragment) BlockData(id int) (bitmapIDs, profileIDs []uint64) {
func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, err error) {
// Ensure that all pair sets are of equal length.
for i := range data {
if len(data[i].BitmapIDs) != len(data[i].ProfileIDs) {
return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].BitmapIDs), len(data[i].ProfileIDs))
if len(data[i].RowIDs) != len(data[i].ColumnIDs) {
return nil, nil, fmt.Errorf("pair set mismatch(idx=%d): %d != %d", i, len(data[i].RowIDs), len(data[i].ColumnIDs))
}
}
@ -801,22 +800,22 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
sets = make([]PairSet, len(data)+1)
clears = make([]PairSet, len(data)+1)
// Limit upper bitmap/profile pair.
maxBitmapID := uint64(id+1) * HashBlockSize
maxProfileID := uint64(SliceWidth)
// Limit upper row/column pair.
maxRowID := uint64(id+1) * HashBlockSize
maxColumnID := uint64(SliceWidth)
// Create buffered iterator for local block.
itrs := make([]*BufIterator, 1, len(data)+1)
itrs[0] = NewBufIterator(
NewLimitIterator(
NewRoaringIterator(f.storage.Iterator()), maxBitmapID, maxProfileID,
NewRoaringIterator(f.storage.Iterator()), maxRowID, maxColumnID,
),
)
// Append buffered iterators for each incoming block.
for i := range data {
var itr Iterator = NewSliceIterator(data[i].BitmapIDs, data[i].ProfileIDs)
itr = NewLimitIterator(itr, maxBitmapID, maxProfileID)
var itr Iterator = NewSliceIterator(data[i].RowIDs, data[i].ColumnIDs)
itr = NewLimitIterator(itr, maxRowID, maxColumnID)
itrs = append(itrs, NewBufIterator(itr))
}
@ -833,8 +832,8 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
values := make([]bool, len(itrs))
for {
var min struct {
bitmapID uint64
profileID uint64
rowID uint64
columnID uint64
}
// Find the lowest pair.
@ -844,9 +843,9 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
if eof { // no more data
continue
} else if !hasData { // first pair
min.bitmapID, min.profileID, hasData = bid, pid, true
} else if bid < min.bitmapID || (bid == min.bitmapID && pid < min.profileID) { // lower pair
min.bitmapID, min.profileID = bid, pid
min.rowID, min.columnID, hasData = bid, pid, true
} else if bid < min.rowID || (bid == min.rowID && pid < min.columnID) { // lower pair
min.rowID, min.columnID = bid, pid
}
}
@ -860,7 +859,7 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
for i, itr := range itrs {
bid, pid, eof := itr.Next()
values[i] = !eof && bid == min.bitmapID && pid == min.profileID
values[i] = !eof && bid == min.rowID && pid == min.columnID
if values[i] {
setN++ // set
} else {
@ -880,25 +879,25 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
// Append to either the set or clear diff.
if newValue {
sets[i].BitmapIDs = append(sets[i].BitmapIDs, min.bitmapID)
sets[i].ProfileIDs = append(sets[i].ProfileIDs, min.profileID)
sets[i].RowIDs = append(sets[i].RowIDs, min.rowID)
sets[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID)
} else {
clears[i].BitmapIDs = append(sets[i].BitmapIDs, min.bitmapID)
clears[i].ProfileIDs = append(sets[i].ProfileIDs, min.profileID)
clears[i].RowIDs = append(sets[i].RowIDs, min.rowID)
clears[i].ColumnIDs = append(sets[i].ColumnIDs, min.columnID)
}
}
}
// Set local bits.
for i := range sets[0].ProfileIDs {
if _, err := f.setBit(sets[0].BitmapIDs[i], (f.Slice()*SliceWidth)+sets[0].ProfileIDs[i]); err != nil {
for i := range sets[0].ColumnIDs {
if _, err := f.setBit(sets[0].RowIDs[i], (f.Slice()*SliceWidth)+sets[0].ColumnIDs[i]); err != nil {
return nil, nil, err
}
}
// Clear local bits.
for i := range clears[0].ProfileIDs {
if _, err := f.clearBit(clears[0].BitmapIDs[i], (f.Slice()*SliceWidth)+clears[0].ProfileIDs[i]); err != nil {
for i := range clears[0].ColumnIDs {
if _, err := f.clearBit(clears[0].RowIDs[i], (f.Slice()*SliceWidth)+clears[0].ColumnIDs[i]); err != nil {
return nil, nil, err
}
}
@ -908,12 +907,12 @@ func (f *Fragment) MergeBlock(id int, data []PairSet) (sets, clears []PairSet, e
// Import bulk imports a set of bits and then snapshots the storage.
// This does not affect the fragment's cache.
func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
func (f *Fragment) Import(rowIDs, columnIDs []uint64) error {
f.mu.Lock()
defer f.mu.Unlock()
// Verify that there are an equal number of bitmap ids and profile ids.
if len(bitmapIDs) != len(profileIDs) {
return fmt.Errorf("mismatch of bitmap/profile len: %d != %d", len(bitmapIDs), len(profileIDs))
// Verify that there are an equal number of row ids and column ids.
if len(rowIDs) != len(columnIDs) {
return fmt.Errorf("mismatch of row/column len: %d != %d", len(rowIDs), len(columnIDs))
}
// Disconnect op writer so we don't append updates.
@ -924,11 +923,11 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
lastID := uint64(0)
if err := func() error {
set := make(map[uint64]struct{})
for i := range bitmapIDs {
bitmapID, profileID := bitmapIDs[i], profileIDs[i]
for i := range rowIDs {
rowID, columnID := rowIDs[i], columnIDs[i]
// Determine the position of the bit in the storage.
pos, err := f.pos(bitmapID, profileID)
pos, err := f.pos(rowID, columnID)
if err != nil {
return err
}
@ -942,21 +941,21 @@ func (f *Fragment) Import(bitmapIDs, profileIDs []uint64) error {
// import optimization to avoid linear foreach calls
// slight risk of concurrent cache counter being off but
// no real danger
if i == 0 || bitmapID != lastID {
lastID = bitmapID
set[bitmapID] = struct{}{}
if i == 0 || rowID != lastID {
lastID = rowID
set[rowID] = struct{}{}
}
// Invalidate block checksum.
delete(f.checksums, int(bitmapID/HashBlockSize))
delete(f.checksums, int(rowID/HashBlockSize))
}
// Update cache counts for all bitmaps.
for bitmapID := range set {
// Import should ALWAYS have bitmap() load a new bm from fragment.storage
// because the bitmap that's in bitmapCache hasn't been updated with
// Update cache counts for all rows.
for rowID := range set {
// Import should ALWAYS have row() load a new bm from fragment.storage
// because the row that's in rowCache hasn't been updated with
// this import's data.
f.cache.BulkAdd(bitmapID, f.bitmap(bitmapID, false, false).Count())
f.cache.BulkAdd(rowID, f.row(rowID, false, false).Count())
}
f.cache.Invalidate()
@ -995,6 +994,7 @@ func (f *Fragment) Snapshot() error {
defer f.mu.Unlock()
return f.snapshot()
}
func track(start time.Time, name string, logger *log.Logger) {
elapsed := time.Since(start)
logger.Printf("%s took %s", name, elapsed)
@ -1061,13 +1061,11 @@ func (f *Fragment) flushCache() error {
return nil
}
// Retrieve a list of bitmap ids from the cache.
bitmapIDs := f.cache.BitmapIDs()
// Retrieve a list of row ids from the cache.
ids := f.cache.IDs()
// Marshal cache data to bytes.
buf, err := proto.Marshal(&internal.Cache{
BitmapIDs: bitmapIDs,
})
buf, err := proto.Marshal(&internal.Cache{IDs: ids})
if err != nil {
return err
}
@ -1253,7 +1251,7 @@ func (f *Fragment) readCacheFromArchive(r io.Reader) error {
return nil
}
// FragmentBlock represents info about a subsection of the bitmaps in a block.
// FragmentBlock represents info about a subsection of the rows in a block.
// This is used for comparing data in remote blocks for active anti-entropy.
type FragmentBlock struct {
ID int `json:"id"`
@ -1386,7 +1384,7 @@ func (s *FragmentSyncer) SyncFragment() error {
return nil
}
// syncBlock sends and receives all bitmaps for a given block.
// syncBlock sends and receives all rows for a given block.
// Returns an error if any remote hosts are unreachable.
func (s *FragmentSyncer) syncBlock(id int) error {
f := s.Fragment
@ -1411,14 +1409,14 @@ func (s *FragmentSyncer) syncBlock(id int) error {
clients = append(clients, client)
// Only sync the standard block.
bitmapIDs, profileIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id)
rowIDs, columnIDs, err := client.BlockData(context.Background(), f.DB(), f.Frame(), ViewStandard, f.Slice(), id)
if err != nil {
return err
}
pairSets = append(pairSets, PairSet{
ProfileIDs: profileIDs,
BitmapIDs: bitmapIDs,
ColumnIDs: columnIDs,
RowIDs: rowIDs,
})
}
@ -1438,7 +1436,7 @@ func (s *FragmentSyncer) syncBlock(id int) error {
set, clear := sets[i], clears[i]
// Ignore if there are no differences.
if len(set.ProfileIDs) == 0 && len(clear.ProfileIDs) == 0 {
if len(set.ColumnIDs) == 0 && len(clear.ColumnIDs) == 0 {
continue
}
@ -1446,11 +1444,11 @@ func (s *FragmentSyncer) syncBlock(id int) error {
var buf bytes.Buffer
// Only sync the standard block.
for j := 0; j < len(set.ProfileIDs); j++ {
fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), set.BitmapIDs[j], (f.Slice()*SliceWidth)+set.ProfileIDs[j])
for j := 0; j < len(set.ColumnIDs); j++ {
fmt.Fprintf(&buf, "SetBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), set.RowIDs[j], (f.Slice()*SliceWidth)+set.ColumnIDs[j])
}
for j := 0; j < len(clear.ProfileIDs); j++ {
fmt.Fprintf(&buf, "ClearBit(frame=%q, id=%d, profileID=%d)\n", f.Frame(), clear.BitmapIDs[j], (f.Slice()*SliceWidth)+clear.ProfileIDs[j])
for j := 0; j < len(clear.ColumnIDs); j++ {
fmt.Fprintf(&buf, "ClearBit(frame=%q, id=%d, columnID=%d)\n", f.Frame(), clear.RowIDs[j], (f.Slice()*SliceWidth)+clear.ColumnIDs[j])
}
// Verify sync is not prematurely closing.
@ -1476,10 +1474,10 @@ func madvise(b []byte, advice int) (err error) {
return
}
// PairSet is a list of equal length bitmap and profile id lists.
// PairSet is a list of equal length row and column id lists.
type PairSet struct {
BitmapIDs []uint64
ProfileIDs []uint64
RowIDs []uint64
ColumnIDs []uint64
}
// byteSlicesEqual returns true if all slices are equal.
@ -1496,7 +1494,7 @@ func byteSlicesEqual(a [][]byte) bool {
return true
}
// Pos returns the bitmap position of a bitmap/profile pair.
func Pos(bitmapID, profileID uint64) uint64 {
return (bitmapID * SliceWidth) + (profileID % SliceWidth)
// Pos returns the row position of a row/column pair.
func Pos(rowID, columnID uint64) uint64 {
return (rowID * SliceWidth) + (columnID % SliceWidth)
}

171
fragment_test.go
View file

@ -35,19 +35,19 @@ func TestFragment_SetBit(t *testing.T) {
t.Fatal(err)
}
// Verify counts on bitmaps.
if n := f.Bitmap(120).Count(); n != 2 {
// Verify counts on rows.
if n := f.Row(120).Count(); n != 2 {
t.Fatalf("unexpected count: %d", n)
} else if n := f.Bitmap(121).Count(); n != 1 {
} else if n := f.Row(121).Count(); n != 1 {
t.Fatalf("unexpected count: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if n := f.Bitmap(120).Count(); n != 2 {
} else if n := f.Row(120).Count(); n != 2 {
t.Fatalf("unexpected count (reopen): %d", n)
} else if n := f.Bitmap(121).Count(); n != 1 {
} else if n := f.Row(121).Count(); n != 1 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
@ -66,15 +66,15 @@ func TestFragment_ClearBit(t *testing.T) {
t.Fatal(err)
}
// Verify count on bitmap.
if n := f.Bitmap(1000).Count(); n != 1 {
// Verify count on row.
if n := f.Row(1000).Count(); n != 1 {
t.Fatalf("unexpected count: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if n := f.Bitmap(1000).Count(); n != 1 {
} else if n := f.Row(1000).Count(); n != 1 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
@ -96,14 +96,14 @@ func TestFragment_Snapshot(t *testing.T) {
// Snapshot bitmap and verify data.
if err := f.Snapshot(); err != nil {
t.Fatal(err)
} else if n := f.Bitmap(1000).Count(); n != 1 {
} else if n := f.Row(1000).Count(); n != 1 {
t.Fatalf("unexpected count: %d", n)
}
// Close and reopen the fragment & verify the data.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if n := f.Bitmap(1000).Count(); n != 1 {
} else if n := f.Row(1000).Count(); n != 1 {
t.Fatalf("unexpected count (reopen): %d", n)
}
}
@ -124,8 +124,8 @@ func TestFragment_ForEachBit(t *testing.T) {
// Iterate over bits.
var result [][2]uint64
if err := f.ForEachBit(func(bitmapID, profileID uint64) error {
result = append(result, [2]uint64{bitmapID, profileID})
if err := f.ForEachBit(func(rowID, columnID uint64) error {
result = append(result, [2]uint64{rowID, columnID})
return nil
}); err != nil {
t.Fatal(err)
@ -142,12 +142,12 @@ func TestFragment_Top(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the bitmaps 100, 101, & 102.
// Set bits on the rows 100, 101, & 102.
f.MustSetBits(100, 1, 3, 200)
f.MustSetBits(101, 1)
f.MustSetBits(102, 1, 2)
// Retrieve top bitmaps.
// Retrieve top rows.
if pairs, err := f.Top(pilosa.TopOptions{N: 2}); err != nil {
t.Fatal(err)
} else if len(pairs) != 2 {
@ -159,21 +159,21 @@ func TestFragment_Top(t *testing.T) {
}
}
// Ensure a fragment can filter bitmaps when retrieving the top n bitmaps.
// Ensure a fragment can filter rows when retrieving the top n rows.
func TestFragment_Top_Filter(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on the bitmaps 100, 101, & 102.
// Set bits on the rows 100, 101, & 102.
f.MustSetBits(100, 1, 3, 200)
f.MustSetBits(101, 1)
f.MustSetBits(102, 1, 2)
// Assign attributes.
f.BitmapAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)})
f.BitmapAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)})
f.RowAttrStore.SetAttrs(101, map[string]interface{}{"x": uint64(10)})
f.RowAttrStore.SetAttrs(102, map[string]interface{}{"x": uint64(20)})
// Retrieve top bitmaps.
// Retrieve top rows.
if pairs, err := f.Top(pilosa.TopOptions{
N: 2,
FilterField: "x",
@ -189,21 +189,21 @@ func TestFragment_Top_Filter(t *testing.T) {
}
}
// Ensure a fragment can return top bitmaps that intersect with an input bitmap.
// Ensure a fragment can return top rows that intersect with an input row.
func TestFragment_TopN_Intersect(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Create an intersecting input bitmap.
// Create an intersecting input row.
src := pilosa.NewBitmap(1, 2, 3)
// Set bits on various bitmaps.
// Set bits on various rows.
f.MustSetBits(100, 1, 10, 11, 12) // one intersection
f.MustSetBits(101, 1, 2, 3, 4) // three intersections
f.MustSetBits(102, 1, 2, 4, 5, 6) // two intersections
f.MustSetBits(103, 1000, 1001, 1002) // no intersection
// Retrieve top bitmaps.
// Retrieve top rows.
if pairs, err := f.Top(pilosa.TopOptions{N: 3, Src: src}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
@ -215,7 +215,7 @@ func TestFragment_TopN_Intersect(t *testing.T) {
}
}
// Ensure a fragment can return top bitmaps that have many bits set.
// Ensure a fragment can return top rows that have many bits set.
func TestFragment_TopN_Intersect_Large(t *testing.T) {
if testing.Short() {
t.Skip("short mode")
@ -224,20 +224,20 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Create an intersecting input bitmap.
// Create an intersecting input row.
src := pilosa.NewBitmap(
980, 981, 982, 983, 984, 985, 986, 987, 988, 989,
990, 991, 992, 993, 994, 995, 996, 997, 998, 999,
)
// Set bits on bitmaps 0 - 999. Higher bitmaps have higher bit counts.
// Set bits on rows 0 - 999. Higher rows have higher bit counts.
for i := uint64(0); i < 1000; i++ {
for j := uint64(0); j < i; j++ {
f.MustSetBits(i, j)
}
}
// Retrieve top bitmaps.
// Retrieve top rows.
if pairs, err := f.Top(pilosa.TopOptions{N: 10, Src: src}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
@ -256,18 +256,18 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
}
}
// Ensure a fragment can return top bitmaps when specified by ID.
func TestFragment_TopN_BitmapIDs(t *testing.T) {
// Ensure a fragment can return top rows when specified by ID.
func TestFragment_TopN_IDs(t *testing.T) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Set bits on various bitmaps.
// Set bits on various rows.
f.MustSetBits(100, 1, 2, 3)
f.MustSetBits(101, 4, 5, 6, 7)
f.MustSetBits(102, 8, 9, 10, 11, 12)
// Retrieve top bitmaps.
if pairs, err := f.Top(pilosa.TopOptions{BitmapIDs: []uint64{100, 101, 200}}); err != nil {
// Retrieve top rows.
if pairs, err := f.Top(pilosa.TopOptions{RowIDs: []uint64{100, 101, 200}}); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(pairs, []pilosa.Pair{
{ID: 101, Count: 4},
@ -307,16 +307,16 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
frag.Close()
f := &Fragment{
Fragment: frag,
BitmapAttrStore: MustOpenAttrStore(),
Fragment: frag,
RowAttrStore: MustOpenAttrStore(),
}
f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
if err := f.Open(); err != nil {
panic(err)
}
defer f.Close()
// Set bits on various bitmaps.
// Set bits on various rows.
f.MustSetBits(100, 1, 2, 3)
f.MustSetBits(101, 4, 5, 6, 7)
f.MustSetBits(102, 8, 9, 10, 11, 12)
@ -332,7 +332,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
{ID: 102, Count: 5},
}
// Retrieve top bitmaps.
// Retrieve top rows.
if pairs, err := f.Top(pilosa.TopOptions{N: 5}); err != nil {
t.Fatal(err)
} else if len(pairs) > int(cacheSize) {
@ -381,7 +381,7 @@ func TestFragment_Blocks(t *testing.T) {
}
prev = blocks
// Set bit on different bitmap.
// Set bit on different row.
if _, err := f.SetBit(20, 0); err != nil {
t.Fatal(err)
}
@ -391,7 +391,7 @@ func TestFragment_Blocks(t *testing.T) {
}
prev = blocks
// Set bit on different profile.
// Set bit on different column.
if _, err := f.SetBit(20, 100); err != nil {
t.Fatal(err)
}
@ -544,7 +544,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
}
// Verify data in other fragment.
if a := f1.Bitmap(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected bits: %+v", a)
}
@ -553,40 +553,11 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
t.Fatal(err)
} else if n := f1.Cache().Len(); n != 1 {
t.Fatalf("unexpected cache size (reopen): %d", n)
} else if a := f1.Bitmap(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
} else if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected bits (reopen): %+v", a)
}
}
/*
func BenchmarkFragment_BlockChecksum_Fill1(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.01) }
func BenchmarkFragment_BlockChecksum_Fill10(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.10) }
func BenchmarkFragment_BlockChecksum_Fill50(b *testing.B) { benchmarkFragmentBlockChecksum(b, 0.50) }
func benchmarkFragmentBlockChecksum(b *testing.B, fillPercent float64) {
f := MustOpenFragment("d", "f", pilosa.ViewStandard, 0)
defer f.Close()
// Fill fragment.
bitmapIDs, profileIDs := GenerateImportFill(pilosa.HashBlockSize, fillPercent)
if err := f.Import(bitmapIDs, profileIDs); err != nil {
b.Fatal(err)
}
b.ResetTimer()
b.ReportAllocs()
// Calculate block checksum.
for i := 0; i < b.N; i++ {
f.InvalidateChecksums()
if chksum := f.BlockChecksum(0); chksum == nil {
b.Fatal("expected checksum")
}
}
}
*/
func BenchmarkFragment_Blocks(b *testing.B) {
if *FragmentPath == "" {
b.Skip("no fragment specified")
@ -633,7 +604,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
// Start benchmark
b.ResetTimer()
for i := 0; i < b.N; i++ {
if n := f.Bitmap(1).IntersectionCount(f.Bitmap(2)); n == 0 {
if n := f.Row(1).IntersectionCount(f.Row(2)); n == 0 {
b.Fatalf("unexpected count: %d", n)
}
}
@ -642,7 +613,7 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
// Fragment is a test wrapper for pilosa.Fragment.
type Fragment struct {
*pilosa.Fragment
BitmapAttrStore *AttrStore
RowAttrStore *AttrStore
}
// NewFragment returns a new instance of Fragment with a temporary path.
@ -654,10 +625,10 @@ func NewFragment(db, frame, view string, slice uint64) *Fragment {
file.Close()
f := &Fragment{
Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice),
BitmapAttrStore: MustOpenAttrStore(),
Fragment: pilosa.NewFragment(file.Name(), db, frame, view, slice),
RowAttrStore: MustOpenAttrStore(),
}
f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
return f
}
@ -674,7 +645,7 @@ func MustOpenFragment(db, frame, view string, slice uint64) *Fragment {
func (f *Fragment) Close() error {
defer os.Remove(f.Path())
defer os.Remove(f.CachePath())
defer f.BitmapAttrStore.Close()
defer f.RowAttrStore.Close()
return f.Fragment.Close()
}
@ -686,64 +657,64 @@ func (f *Fragment) Reopen() error {
}
f.Fragment = pilosa.NewFragment(path, f.DB(), f.Frame(), f.View(), f.Slice())
f.Fragment.BitmapAttrStore = f.BitmapAttrStore.AttrStore
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
if err := f.Open(); err != nil {
return err
}
return nil
}
// MustSetBits sets bits on a bitmap. Panic on error.
// MustSetBits sets bits on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(bitmapID uint64, profileIDs ...uint64) {
for _, profileID := range profileIDs {
if _, err := f.SetBit(bitmapID, profileID); err != nil {
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.SetBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// MustClearBits clears bits on a bitmap. Panic on error.
func (f *Fragment) MustClearBits(bitmapID uint64, profileIDs ...uint64) {
for _, profileID := range profileIDs {
if _, err := f.ClearBit(bitmapID, profileID); err != nil {
// MustClearBits clears bits on a row. Panic on error.
func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.ClearBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// BitmapAttrStore provides simple storage for attributes.
type BitmapAttrStore struct {
// RowAttrStore provides simple storage for attributes.
type RowAttrStore struct {
attrs map[uint64]map[string]interface{}
}
// NewBitmapAttrStore returns a new instance of BitmapAttrStore.
func NewBitmapAttrStore() *BitmapAttrStore {
return &BitmapAttrStore{
// NewRowAttrStore returns a new instance of RowAttrStore.
func NewRowAttrStore() *RowAttrStore {
return &RowAttrStore{
attrs: make(map[uint64]map[string]interface{}),
}
}
// BitmapAttrs returns the attributes set to a bitmap id.
func (s *BitmapAttrStore) BitmapAttrs(id uint64) (map[string]interface{}, error) {
// RowAttrs returns the attributes set to a row id.
func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) {
return s.attrs[id], nil
}
// SetBitmapAttrs assigns a set of attributes to a bitmap id.
func (s *BitmapAttrStore) SetBitmapAttrs(id uint64, m map[string]interface{}) {
// SetRowAttrs assigns a set of attributes to a row id.
func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) {
s.attrs[id] = m
}
// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk.
func GenerateImportFill(bitmapN int, pct float64) (bitmapIDs, profileIDs []uint64) {
func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
ipct := int(pct * 100)
for i := 0; i < SliceWidth*bitmapN; i++ {
for i := 0; i < SliceWidth*rowN; i++ {
if i%100 >= ipct {
continue
}
bitmapIDs = append(bitmapIDs, uint64(i%SliceWidth))
profileIDs = append(profileIDs, uint64(i/SliceWidth))
rowIDs = append(rowIDs, uint64(i%SliceWidth))
columnIDs = append(columnIDs, uint64(i/SliceWidth))
}
return
}
@ -754,7 +725,7 @@ func TestFragment_Tanimoto(t *testing.T) {
src := pilosa.NewBitmap(1, 2, 3)
// Set bits on the bitmaps 100, 101, & 102.
// Set bits on the rows 100, 101, & 102.
f.MustSetBits(100, 1, 3, 2, 200)
f.MustSetBits(101, 1, 3)
f.MustSetBits(102, 1, 2, 10, 12)
@ -776,7 +747,7 @@ func TestFragment_Zero_Tanimoto(t *testing.T) {
src := pilosa.NewBitmap(1, 2, 3)
// Set bits on the bitmaps 100, 101, & 102.
// Set bits on the rows 100, 101, & 102.
f.MustSetBits(100, 1, 3, 2, 200)
f.MustSetBits(101, 1, 3)
f.MustSetBits(102, 1, 2, 10, 12)

58
frame.go
View file

@ -35,8 +35,8 @@ type Frame struct {
views map[string]*View
// Bitmap attribute storage and cache
bitmapAttrStore *AttrStore
// Row attribute storage and cache
rowAttrStore *AttrStore
broadcaster Broadcaster
stats StatsClient
@ -64,8 +64,8 @@ func NewFrame(path, db, name string) (*Frame, error) {
db: db,
name: name,
views: make(map[string]*View),
bitmapAttrStore: NewAttrStore(filepath.Join(path, ".data")),
views: make(map[string]*View),
rowAttrStore: NewAttrStore(filepath.Join(path, ".data")),
stats: NopStatsClient,
@ -87,8 +87,8 @@ func (f *Frame) DB() string { return f.db }
// Path returns the path the frame was initialized with.
func (f *Frame) Path() string { return f.path }
// BitmapAttrStore returns the attribute storage.
func (f *Frame) BitmapAttrStore() *AttrStore { return f.bitmapAttrStore }
// RowAttrStore returns the attribute storage.
func (f *Frame) RowAttrStore() *AttrStore { return f.rowAttrStore }
// MaxSlice returns the max slice in the frame.
func (f *Frame) MaxSlice() uint64 {
@ -215,7 +215,7 @@ func (f *Frame) Open() error {
return err
}
if err := f.bitmapAttrStore.Open(); err != nil {
if err := f.rowAttrStore.Open(); err != nil {
return err
}
@ -253,7 +253,7 @@ func (f *Frame) openViews() error {
if err := view.Open(); err != nil {
return fmt.Errorf("open view: view=%s, err=%s", view.Name(), err)
}
view.BitmapAttrStore = f.bitmapAttrStore
view.RowAttrStore = f.rowAttrStore
f.views[view.Name()] = view
f.stats.Count("maxSlice", 1)
@ -326,8 +326,8 @@ func (f *Frame) Close() error {
defer f.mu.Unlock()
// Close the attribute store.
if f.bitmapAttrStore != nil {
_ = f.bitmapAttrStore.Close()
if f.rowAttrStore != nil {
_ = f.rowAttrStore.Close()
}
// Close all views.
@ -411,7 +411,7 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
if err := view.Open(); err != nil {
return nil, err
}
view.BitmapAttrStore = f.bitmapAttrStore
view.RowAttrStore = f.rowAttrStore
f.views[view.Name()] = view
return view, nil
@ -421,7 +421,7 @@ func (f *Frame) newView(path, name string) *View {
view := NewView(path, f.db, f.name, name, f.cacheSize)
view.cacheType = f.cacheType
view.LogOutput = f.LogOutput
view.BitmapAttrStore = f.bitmapAttrStore
view.RowAttrStore = f.rowAttrStore
view.stats = f.stats.WithTags(fmt.Sprintf("slice:%s", name))
return view
}
@ -511,7 +511,7 @@ func (f *Frame) ClearBit(name string, rowID, colID uint64, t *time.Time) (change
}
// Import bulk imports data.
func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time) error {
func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) error {
// Determine quantum if timestamps are set.
q := f.TimeQuantum()
if hasTime(timestamps) && q == "" {
@ -520,8 +520,8 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time)
// Split import data by fragment.
dataByFragment := make(map[importKey]importData)
for i := range bitmapIDs {
bitmapID, profileID, timestamp := bitmapIDs[i], profileIDs[i], timestamps[i]
for i := range rowIDs {
rowID, columnID, timestamp := rowIDs[i], columnIDs[i], timestamps[i]
var standard, inverse []string
if timestamp == nil {
@ -534,20 +534,20 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time)
// Attach bit to each standard view.
for _, name := range standard {
key := importKey{View: name, Slice: profileID / SliceWidth}
key := importKey{View: name, Slice: columnID / SliceWidth}
data := dataByFragment[key]
data.BitmapIDs = append(data.BitmapIDs, bitmapID)
data.ProfileIDs = append(data.ProfileIDs, profileID)
data.RowIDs = append(data.RowIDs, rowID)
data.ColumnIDs = append(data.ColumnIDs, columnID)
dataByFragment[key] = data
}
if f.inverseEnabled {
// Attach reversed bits to each inverse view.
for _, name := range inverse {
key := importKey{View: name, Slice: bitmapID / SliceWidth}
key := importKey{View: name, Slice: rowID / SliceWidth}
data := dataByFragment[key]
data.BitmapIDs = append(data.BitmapIDs, profileID) // reversed
data.ProfileIDs = append(data.ProfileIDs, bitmapID) // reversed
data.RowIDs = append(data.RowIDs, columnID) // reversed
data.ColumnIDs = append(data.ColumnIDs, rowID) // reversed
dataByFragment[key] = data
}
}
@ -563,8 +563,8 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time)
// Re-sort data for inverse views.
if IsInverseView(key.View) {
sort.Sort(importBitSet{
bitmapIDs: data.BitmapIDs,
profileIDs: data.ProfileIDs,
rowIDs: data.RowIDs,
columnIDs: data.ColumnIDs,
})
}
@ -578,7 +578,7 @@ func (f *Frame) Import(bitmapIDs, profileIDs []uint64, timestamps []*time.Time)
return err
}
if err := frag.Import(data.BitmapIDs, data.ProfileIDs); err != nil {
if err := frag.Import(data.RowIDs, data.ColumnIDs); err != nil {
return err
}
}
@ -650,15 +650,15 @@ func (o *FrameOptions) Encode() *internal.FrameMeta {
// importBitSet represents slices of row and column ids.
// This is used to sort data during import.
type importBitSet struct {
bitmapIDs, profileIDs []uint64
rowIDs, columnIDs []uint64
}
func (p importBitSet) Swap(i, j int) {
p.bitmapIDs[i], p.bitmapIDs[j] = p.bitmapIDs[j], p.bitmapIDs[i]
p.profileIDs[i], p.profileIDs[j] = p.profileIDs[j], p.profileIDs[i]
p.rowIDs[i], p.rowIDs[j] = p.rowIDs[j], p.rowIDs[i]
p.columnIDs[i], p.columnIDs[j] = p.columnIDs[j], p.columnIDs[i]
}
func (p importBitSet) Len() int { return len(p.bitmapIDs) }
func (p importBitSet) Less(i, j int) bool { return p.bitmapIDs[i] < p.bitmapIDs[j] }
func (p importBitSet) Len() int { return len(p.rowIDs) }
func (p importBitSet) Less(i, j int) bool { return p.rowIDs[i] < p.rowIDs[j] }
// Cache types.
const (

4
frame_test.go
View file

@ -119,8 +119,8 @@ func (f *Frame) Reopen() error {
}
// MustSetBit sets a bit on the frame. Panic on error.
func (f *Frame) MustSetBit(view string, bitmapID, profileID uint64, t *time.Time) (changed bool) {
changed, err := f.SetBit(view, bitmapID, profileID, t)
func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) {
changed, err := f.SetBit(view, rowID, columnID, t)
if err != nil {
panic(err)
}

88
handler.go
View file

@ -159,26 +159,26 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
results, err := h.Executor.Execute(r.Context(), dbName, q, req.Slices, opt)
resp := &QueryResponse{Results: results, Err: err}
// Fill profile attributes if requested.
if req.Profiles {
// Consolidate all profile ids across all calls.
var profileIDs []uint64
// Fill column attributes if requested.
if req.ColumnAttrs {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
bm, ok := result.(*Bitmap)
if !ok {
continue
}
profileIDs = uint64Slice(profileIDs).merge(bm.Bits())
columnIDs = uint64Slice(columnIDs).merge(bm.Bits())
}
// Retrieve profile attributes across all calls.
profiles, err := h.readProfiles(h.Index.DB(dbName), profileIDs)
// Retrieve column attributes across all calls.
columnAttrSets, err := h.readColumnAttrSets(h.Index.DB(dbName), columnIDs)
if err != nil {
w.WriteHeader(http.StatusInternalServerError)
h.writeQueryResponse(w, r, &QueryResponse{Err: err})
return
}
resp.Profiles = profiles
resp.ColumnAttrSets = columnAttrSets
}
// Set appropriate status code, if there is an error.
@ -437,7 +437,7 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) {
}
// Retrieve local blocks.
blks, err := db.ProfileAttrStore().Blocks()
blks, err := db.ColumnAttrStore().Blocks()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -447,7 +447,7 @@ func (h *Handler) handlePostDBAttrDiff(w http.ResponseWriter, r *http.Request) {
attrs := make(map[uint64]map[string]interface{})
for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) {
// Retrieve block data.
m, err := db.ProfileAttrStore().BlockData(blockID)
m, err := db.ColumnAttrStore().BlockData(blockID)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -701,7 +701,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request
}
// Retrieve local blocks.
blks, err := f.BitmapAttrStore().Blocks()
blks, err := f.RowAttrStore().Blocks()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -711,7 +711,7 @@ func (h *Handler) handlePostFrameAttrDiff(w http.ResponseWriter, r *http.Request
attrs := make(map[uint64]map[string]interface{})
for _, blockID := range AttrBlocks(blks).Diff(req.Blocks) {
// Retrieve block data.
m, err := f.BitmapAttrStore().BlockData(blockID)
m, err := f.RowAttrStore().BlockData(blockID)
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
@ -739,24 +739,24 @@ type postFrameAttrDiffResponse struct {
Attrs map[uint64]map[string]interface{} `json:"attrs"`
}
// readProfiles returns a list of profile objects by id.
func (h *Handler) readProfiles(db *DB, ids []uint64) ([]*Profile, error) {
// readColumnAttrSets returns a list of column attribute objects by id.
func (h *Handler) readColumnAttrSets(db *DB, ids []uint64) ([]*ColumnAttrSet, error) {
if db == nil {
return nil, nil
}
a := make([]*Profile, 0, len(ids))
a := make([]*ColumnAttrSet, 0, len(ids))
for _, id := range ids {
// Read attributes for profile. Skip profile if empty.
attrs, err := db.ProfileAttrStore().Attrs(id)
// Read attributes for column. Skip column if empty.
attrs, err := db.ColumnAttrStore().Attrs(id)
if err != nil {
return nil, err
} else if len(attrs) == 0 {
continue
}
// Append profile with attributes.
a = append(a, &Profile{ID: id, Attrs: attrs})
// Append column with attributes.
a = append(a, &ColumnAttrSet{ID: id, Attrs: attrs})
}
return a, nil
@ -817,10 +817,10 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
}
return &QueryRequest{
Query: query,
Slices: slices,
Profiles: q.Get("profiles") == "true",
Quantum: quantum,
Query: query,
Slices: slices,
ColumnAttrs: q.Get("columnAttrs") == "true",
Quantum: quantum,
}, nil
}
@ -907,9 +907,9 @@ func (h *Handler) handlePostImport(w http.ResponseWriter, r *http.Request) {
}
// Import into fragment.
err = f.Import(req.BitmapIDs, req.ProfileIDs, timestamps)
err = f.Import(req.RowIDs, req.ColumnIDs, timestamps)
if err != nil {
h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ProfileIDs), err)
h.logger().Printf("import error: db=%s, frame=%s, slice=%d, bits=%d, err=%s", req.DB, req.Frame, req.Slice, len(req.ColumnIDs), err)
return
}
@ -965,10 +965,10 @@ func (h *Handler) handleGetExportCSV(w http.ResponseWriter, r *http.Request) {
cw := csv.NewWriter(w)
// Iterate over each bit.
if err := f.ForEachBit(func(bitmapID, profileID uint64) error {
if err := f.ForEachBit(func(rowID, columnID uint64) error {
return cw.Write([]string{
strconv.FormatUint(bitmapID, 10),
strconv.FormatUint(profileID, 10),
strconv.FormatUint(rowID, 10),
strconv.FormatUint(columnID, 10),
})
}); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
@ -1083,7 +1083,7 @@ func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Requ
// Read data
var resp internal.BlockDataResponse
if f != nil {
resp.BitmapIDs, resp.ProfileIDs = f.BlockData(int(req.Block))
resp.RowIDs, resp.ColumnIDs = f.BlockData(int(req.Block))
}
// Encode response.
@ -1271,8 +1271,8 @@ type QueryRequest struct {
// If empty, all slices are included.
Slices []uint64
// Return profile attributes, if true.
Profiles bool
// Return column attributes, if true.
ColumnAttrs bool
// Time granularity to use with the timestamp.
Quantum TimeQuantum
@ -1284,11 +1284,11 @@ type QueryRequest struct {
func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest {
req := &QueryRequest{
Query: pb.Query,
Slices: pb.Slices,
Profiles: pb.Profiles,
Quantum: TimeQuantum(pb.Quantum),
Remote: pb.Remote,
Query: pb.Query,
Slices: pb.Slices,
ColumnAttrs: pb.ColumnAttrs,
Quantum: TimeQuantum(pb.Quantum),
Remote: pb.Remote,
}
return req
@ -1300,8 +1300,8 @@ type QueryResponse struct {
// Can be a Bitmap, Pairs, or uint64.
Results []interface{}
// Set of profiles matching IDs returned in Result.
Profiles []*Profile
// Set of column attribute objects matching IDs returned in Result.
ColumnAttrSets []*ColumnAttrSet
// Error during parsing or execution.
Err error
@ -1309,12 +1309,12 @@ type QueryResponse struct {
func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
var output struct {
Results []interface{} `json:"results,omitempty"`
Profiles []*Profile `json:"profiles,omitempty"`
Err string `json:"error,omitempty"`
Results []interface{} `json:"results,omitempty"`
ColumnAttrSets []*ColumnAttrSet `json:"columnAttrs,omitempty"`
Err string `json:"error,omitempty"`
}
output.Results = resp.Results
output.Profiles = resp.Profiles
output.ColumnAttrSets = resp.ColumnAttrSets
if resp.Err != nil {
output.Err = resp.Err.Error()
@ -1324,8 +1324,8 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
func encodeQueryResponse(resp *QueryResponse) *internal.QueryResponse {
pb := &internal.QueryResponse{
Results: make([]*internal.QueryResult, len(resp.Results)),
Profiles: encodeProfiles(resp.Profiles),
Results: make([]*internal.QueryResult, len(resp.Results)),
ColumnAttrSets: encodeColumnAttrSets(resp.ColumnAttrSets),
}
for i := range resp.Results {

50
handler_test.go
View file

@ -253,18 +253,18 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
}
}
// Ensure the handler can execute a query that returns a bitmap with profiles as JSON.
func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap with column attributes as JSON.
func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
idx := NewIndex()
defer idx.Close()
// Create database and set profile attributes.
// Create database and set column attributes.
db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
if err != nil {
t.Fatal(err)
} else if err := db.ProfileAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
} else if err := db.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := db.ProfileAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil {
} else if err := db.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil {
t.Fatal(err)
}
@ -277,10 +277,10 @@ func TestHandler_Query_Bitmap_Profiles_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query?profiles=true", strings.NewReader("Bitmap(id=100)")))
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/db/d/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"profiles":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -318,16 +318,16 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
}
}
// Ensure the handler can execute a query that returns a bitmap with profiles as protobuf.
func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap with column attributes as protobuf.
func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
idx := NewIndex()
defer idx.Close()
// Create database and set profile attributes.
// Create database and set column attributes.
db, err := idx.CreateDBIfNotExists("d", pilosa.DBOptions{})
if err != nil {
t.Fatal(err)
} else if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
} else if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
}
@ -341,8 +341,8 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
// Encode request body.
buf, err := proto.Marshal(&internal.QueryRequest{
Query: "Bitmap(id=100)",
Profiles: true,
Query: "Bitmap(id=100)",
ColumnAttrs: true,
})
if err != nil {
t.Fatal(err)
@ -373,12 +373,12 @@ func TestHandler_Query_Bitmap_Profiles_Protobuf(t *testing.T) {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
}
if a := resp.Profiles; len(a) != 1 {
t.Fatalf("unexpected profiles length: %d", len(a))
if a := resp.ColumnAttrSets; len(a) != 1 {
t.Fatalf("unexpected column attributes length: %d", len(a))
} else if a[0].ID != 1 {
t.Fatalf("unexpected id: %d", a[0].ID)
} else if len(a[0].Attrs) != 1 {
t.Fatalf("unexpected profile attr length: %d", len(a))
t.Fatalf("unexpected column attr length: %d", len(a))
} else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
}
@ -603,16 +603,16 @@ func TestHandler_DB_AttrStore_Diff(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if err := db.ProfileAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
if err := db.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := db.ProfileAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
} else if err := db.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := db.ProfileAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
} else if err := db.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
// Retrieve block checksums.
blks, err := db.ProfileAttrStore().Blocks()
blks, err := db.ColumnAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
@ -653,16 +653,16 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
if err != nil {
t.Fatal(err)
}
if err := f.BitmapAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := f.BitmapAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
} else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := f.BitmapAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
} else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
// Retrieve block checksums.
blks, err := f.BitmapAttrStore().Blocks()
blks, err := f.RowAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
@ -732,7 +732,7 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
f1 := idx.Fragment("x", "y", pilosa.ViewStandard, 0)
if f1 == nil {
t.Fatal("fragment x/y/standard/0 not created")
} else if bits := f1.Bitmap(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) {
} else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) {
t.Fatalf("unexpected restored bits: %+v", bits)
}
}

22
index.go
View file

@ -34,7 +34,7 @@ type Index struct {
// Data directory path.
Path string
// The interval at which the cached bitmap ids are persisted to disk.
// The interval at which the cached row ids are persisted to disk.
CacheFlushInterval time.Duration
LogOutput io.Writer
@ -375,7 +375,7 @@ func (s *IndexSyncer) SyncIndex() error {
return nil
}
// Sync database profile attributes.
// Sync database column attributes.
if err := s.syncDatabase(di.Name); err != nil {
return fmt.Errorf("db sync error: db=%s, err=%s", di.Name, err)
}
@ -386,7 +386,7 @@ func (s *IndexSyncer) SyncIndex() error {
return nil
}
// Sync frame bitmap attributes.
// Sync frame row attributes.
if err := s.syncFrame(di.Name, fi.Name); err != nil {
return fmt.Errorf("frame sync error: db=%s, frame=%s, err=%s", di.Name, fi.Name, err)
}
@ -429,7 +429,7 @@ func (s *IndexSyncer) syncDatabase(db string) error {
}
// Read block checksums.
blks, err := d.ProfileAttrStore().Blocks()
blks, err := d.ColumnAttrStore().Blocks()
if err != nil {
return err
}
@ -443,7 +443,7 @@ func (s *IndexSyncer) syncDatabase(db string) error {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := client.ProfileAttrDiff(context.Background(), db, blks)
m, err := client.ColumnAttrDiff(context.Background(), db, blks)
if err != nil {
return err
} else if len(m) == 0 {
@ -451,12 +451,12 @@ func (s *IndexSyncer) syncDatabase(db string) error {
}
// Update local copy.
if err := d.ProfileAttrStore().SetBulkAttrs(m); err != nil {
if err := d.ColumnAttrStore().SetBulkAttrs(m); err != nil {
return err
}
// Recompute blocks.
blks, err = d.ProfileAttrStore().Blocks()
blks, err = d.ColumnAttrStore().Blocks()
if err != nil {
return err
}
@ -474,7 +474,7 @@ func (s *IndexSyncer) syncFrame(db, name string) error {
}
// Read block checksums.
blks, err := f.BitmapAttrStore().Blocks()
blks, err := f.RowAttrStore().Blocks()
if err != nil {
return err
}
@ -488,7 +488,7 @@ func (s *IndexSyncer) syncFrame(db, name string) error {
// Retrieve attributes from differing blocks.
// Skip update and recomputation if no attributes have changed.
m, err := client.BitmapAttrDiff(context.Background(), db, name, blks)
m, err := client.RowAttrDiff(context.Background(), db, name, blks)
if err == ErrFrameNotFound {
continue // frame not created remotely yet, skip
} else if err != nil {
@ -498,12 +498,12 @@ func (s *IndexSyncer) syncFrame(db, name string) error {
}
// Update local copy.
if err := f.BitmapAttrStore().SetBulkAttrs(m); err != nil {
if err := f.RowAttrStore().SetBulkAttrs(m); err != nil {
return err
}
// Recompute blocks.
blks, err = f.BitmapAttrStore().Blocks()
blks, err = f.RowAttrStore().Blocks()
if err != nil {
return err
}

16
index_test.go
View file

@ -135,28 +135,28 @@ func TestIndexSyncer_SyncIndex(t *testing.T) {
// Verify data is the same on both nodes.
for i, idx := range []*Index{idx0, idx1} {
f := idx.Fragment("d", "f", pilosa.ViewStandard, 0)
if a := f.Bitmap(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Bitmap(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Bitmap(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Bitmap(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Bitmap(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
}
f = idx.Fragment("d", "f0", pilosa.ViewStandard, 1)
a := f.Bitmap(9).Bits()
a := f.Row(9).Bits()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
}
if a := f.Bitmap(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
}
f = idx.Fragment("y", "z", pilosa.ViewStandard, 3)
if a := f.Bitmap(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
}
}

132
internal/private.pb.go
View file

@ -90,8 +90,8 @@ func (*BlockDataRequest) ProtoMessage() {}
func (*BlockDataRequest) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{3} }
type BlockDataResponse struct {
BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"`
ProfileIDs []uint64 `protobuf:"varint,2,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"`
RowIDs []uint64 `protobuf:"varint,1,rep,packed,name=RowIDs" json:"RowIDs,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,2,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
}
func (m *BlockDataResponse) Reset() { *m = BlockDataResponse{} }
@ -100,7 +100,7 @@ func (*BlockDataResponse) ProtoMessage() {}
func (*BlockDataResponse) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{4} }
type Cache struct {
BitmapIDs []uint64 `protobuf:"varint,1,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"`
IDs []uint64 `protobuf:"varint,1,rep,packed,name=IDs" json:"IDs,omitempty"`
}
func (m *Cache) Reset() { *m = Cache{} }
@ -432,10 +432,10 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.BitmapIDs) > 0 {
dAtA2 := make([]byte, len(m.BitmapIDs)*10)
if len(m.RowIDs) > 0 {
dAtA2 := make([]byte, len(m.RowIDs)*10)
var j1 int
for _, num := range m.BitmapIDs {
for _, num := range m.RowIDs {
for num >= 1<<7 {
dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -449,10 +449,10 @@ func (m *BlockDataResponse) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPrivate(dAtA, i, uint64(j1))
i += copy(dAtA[i:], dAtA2[:j1])
}
if len(m.ProfileIDs) > 0 {
dAtA4 := make([]byte, len(m.ProfileIDs)*10)
if len(m.ColumnIDs) > 0 {
dAtA4 := make([]byte, len(m.ColumnIDs)*10)
var j3 int
for _, num := range m.ProfileIDs {
for _, num := range m.ColumnIDs {
for num >= 1<<7 {
dAtA4[j3] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -484,10 +484,10 @@ func (m *Cache) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.BitmapIDs) > 0 {
dAtA6 := make([]byte, len(m.BitmapIDs)*10)
if len(m.IDs) > 0 {
dAtA6 := make([]byte, len(m.IDs)*10)
var j5 int
for _, num := range m.BitmapIDs {
for _, num := range m.IDs {
for num >= 1<<7 {
dAtA6[j5] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -924,16 +924,16 @@ func (m *BlockDataRequest) Size() (n int) {
func (m *BlockDataResponse) Size() (n int) {
var l int
_ = l
if len(m.BitmapIDs) > 0 {
if len(m.RowIDs) > 0 {
l = 0
for _, e := range m.BitmapIDs {
for _, e := range m.RowIDs {
l += sovPrivate(uint64(e))
}
n += 1 + sovPrivate(uint64(l)) + l
}
if len(m.ProfileIDs) > 0 {
if len(m.ColumnIDs) > 0 {
l = 0
for _, e := range m.ProfileIDs {
for _, e := range m.ColumnIDs {
l += sovPrivate(uint64(e))
}
n += 1 + sovPrivate(uint64(l)) + l
@ -944,9 +944,9 @@ func (m *BlockDataResponse) Size() (n int) {
func (m *Cache) Size() (n int) {
var l int
_ = l
if len(m.BitmapIDs) > 0 {
if len(m.IDs) > 0 {
l = 0
for _, e := range m.BitmapIDs {
for _, e := range m.IDs {
l += sovPrivate(uint64(e))
}
n += 1 + sovPrivate(uint64(l)) + l
@ -1714,7 +1714,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error {
break
}
}
m.BitmapIDs = append(m.BitmapIDs, v)
m.RowIDs = append(m.RowIDs, v)
}
} else if wireType == 0 {
var v uint64
@ -1732,9 +1732,9 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error {
break
}
}
m.BitmapIDs = append(m.BitmapIDs, v)
m.RowIDs = append(m.RowIDs, v)
} else {
return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType)
}
case 2:
if wireType == 2 {
@ -1776,7 +1776,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error {
break
}
}
m.ProfileIDs = append(m.ProfileIDs, v)
m.ColumnIDs = append(m.ColumnIDs, v)
}
} else if wireType == 0 {
var v uint64
@ -1794,9 +1794,9 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error {
break
}
}
m.ProfileIDs = append(m.ProfileIDs, v)
m.ColumnIDs = append(m.ColumnIDs, v)
} else {
return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType)
}
default:
iNdEx = preIndex
@ -1888,7 +1888,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error {
break
}
}
m.BitmapIDs = append(m.BitmapIDs, v)
m.IDs = append(m.IDs, v)
}
} else if wireType == 0 {
var v uint64
@ -1906,9 +1906,9 @@ func (m *Cache) Unmarshal(dAtA []byte) error {
break
}
}
m.BitmapIDs = append(m.BitmapIDs, v)
m.IDs = append(m.IDs, v)
} else {
return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field IDs", wireType)
}
default:
iNdEx = preIndex
@ -3146,43 +3146,43 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 600 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x94, 0x54, 0xcd, 0x4e, 0x14, 0x41,
0x10, 0x76, 0x7e, 0x20, 0x4c, 0x21, 0xcb, 0xd2, 0x7a, 0x98, 0x10, 0x32, 0x59, 0x3b, 0x2a, 0xc4,
0x03, 0x07, 0xbc, 0x18, 0xe2, 0x69, 0x18, 0x14, 0x12, 0x20, 0xd2, 0x8b, 0xde, 0x7b, 0x97, 0x52,
0x27, 0x3b, 0x7f, 0xce, 0xf4, 0x2e, 0xac, 0x57, 0x5f, 0xc2, 0xc4, 0x67, 0xf0, 0x3d, 0x3c, 0xfa,
0x08, 0x66, 0x7d, 0x11, 0xd3, 0xdd, 0xf3, 0xe7, 0xb2, 0xf8, 0x73, 0xeb, 0xfa, 0xaa, 0xea, 0xab,
0xaf, 0xbf, 0xa9, 0x1e, 0x58, 0xcb, 0xf2, 0x70, 0xc2, 0x05, 0xee, 0x66, 0x79, 0x2a, 0x52, 0xb2,
0x12, 0x26, 0x02, 0xf3, 0x84, 0x47, 0xf4, 0x04, 0x96, 0x03, 0xff, 0x14, 0x05, 0x27, 0x3d, 0x58,
0x3d, 0x48, 0xa3, 0x71, 0x9c, 0x9c, 0xf0, 0x01, 0x46, 0xae, 0xd1, 0x33, 0x76, 0x1c, 0xd6, 0x86,
0x64, 0xc5, 0x45, 0x18, 0xe3, 0xf9, 0x98, 0x27, 0x62, 0x1c, 0xbb, 0xa6, 0xae, 0x68, 0x41, 0xf4,
0xab, 0x01, 0xce, 0x8b, 0x9c, 0xc7, 0xa8, 0x18, 0x37, 0x61, 0x85, 0xa5, 0x57, 0x6d, 0xba, 0x3a,
0x26, 0x8f, 0xa1, 0x73, 0x9c, 0x4c, 0x30, 0x2f, 0xf0, 0x30, 0xe1, 0x83, 0x08, 0x2f, 0x15, 0xdd,
0x0a, 0x9b, 0x43, 0xc9, 0x16, 0x38, 0x07, 0x7c, 0xf8, 0x1e, 0x2f, 0xa6, 0x19, 0xba, 0x96, 0x22,
0x69, 0x80, 0x3a, 0xdb, 0x0f, 0x3f, 0xa2, 0x6b, 0xf7, 0x8c, 0x9d, 0x35, 0xd6, 0x00, 0xf3, 0x7a,
0x97, 0x6e, 0xea, 0xa5, 0xd0, 0x39, 0x8e, 0xb3, 0x34, 0x17, 0x0c, 0x8b, 0x2c, 0x4d, 0x0a, 0x24,
0x5d, 0xb0, 0x0e, 0xf3, 0xbc, 0x94, 0x2b, 0x8f, 0xf4, 0x1a, 0xba, 0x7e, 0x94, 0x0e, 0x47, 0x01,
0x17, 0x9c, 0xe1, 0x87, 0x31, 0x16, 0x82, 0x74, 0xc0, 0x0c, 0xfc, 0xb2, 0xc8, 0x0c, 0x7c, 0x72,
0x1f, 0x96, 0xd4, 0xb5, 0x4b, 0x4f, 0x74, 0x20, 0x51, 0xd5, 0xa9, 0x74, 0xdb, 0x4c, 0x07, 0x12,
0xed, 0x47, 0xe1, 0x50, 0xeb, 0xb5, 0x99, 0x0e, 0x08, 0x01, 0xfb, 0x4d, 0x88, 0x57, 0xa5, 0x48,
0x75, 0xa6, 0xe7, 0xb0, 0xd1, 0x9a, 0x5c, 0x0a, 0xdc, 0x02, 0xc7, 0x0f, 0x45, 0xcc, 0xb3, 0xe3,
0xa0, 0x70, 0x8d, 0x9e, 0xb5, 0x63, 0xb3, 0x06, 0x20, 0x1e, 0xc0, 0xab, 0x3c, 0x7d, 0x1b, 0x46,
0x28, 0xd3, 0xa6, 0x4a, 0xb7, 0x10, 0xfa, 0x08, 0x96, 0x94, 0x3f, 0x7f, 0xa6, 0xa1, 0x5f, 0x0c,
0xd8, 0x38, 0xe5, 0xd7, 0x4a, 0x5a, 0x51, 0x8f, 0x3e, 0x02, 0xa7, 0x06, 0x55, 0xcf, 0xea, 0xde,
0x93, 0xdd, 0x6a, 0x93, 0x76, 0x6f, 0xd4, 0x37, 0xc8, 0x61, 0x22, 0xf2, 0x29, 0x6b, 0x9a, 0x37,
0x9f, 0x43, 0xe7, 0xf7, 0xa4, 0xf4, 0x7d, 0x84, 0xd3, 0xca, 0xf7, 0x11, 0x4e, 0xa5, 0x4f, 0x13,
0x1e, 0x8d, 0xb5, 0xa7, 0x36, 0xd3, 0xc1, 0xbe, 0xf9, 0xcc, 0xa0, 0xfb, 0x40, 0x0e, 0x72, 0xe4,
0x02, 0x15, 0xc1, 0x29, 0x16, 0x05, 0x7f, 0x87, 0x8b, 0xbe, 0x89, 0xf6, 0xd9, 0x6c, 0xf9, 0x4c,
0x1f, 0xc0, 0x7a, 0x80, 0x11, 0x0a, 0x94, 0x5b, 0xbf, 0xb0, 0x91, 0xbe, 0x84, 0x75, 0x4d, 0x7f,
0x6b, 0x09, 0x79, 0x08, 0xb6, 0xdc, 0x70, 0x45, 0xbd, 0xba, 0xd7, 0x6d, 0x4c, 0xd0, 0x6f, 0x89,
0xa9, 0x2c, 0x1d, 0x56, 0x3a, 0xcb, 0x27, 0x71, 0xab, 0xce, 0x05, 0xbb, 0xb3, 0x5d, 0x4e, 0xb0,
0xd4, 0x84, 0x7b, 0xcd, 0x84, 0xfa, 0x79, 0x95, 0x43, 0xf6, 0x81, 0xe8, 0x0b, 0xfd, 0xff, 0x10,
0x1a, 0x94, 0xa8, 0xdc, 0xbe, 0x33, 0x99, 0xd5, 0x0d, 0xea, 0x5c, 0x2b, 0x30, 0xff, 0xa6, 0xe0,
0x93, 0x21, 0x87, 0x2d, 0xe4, 0xf8, 0x27, 0x9f, 0xe4, 0x7f, 0xa2, 0xda, 0x86, 0xf2, 0xa9, 0xd4,
0x31, 0xd9, 0x86, 0x65, 0x35, 0xaf, 0x70, 0x6d, 0xb5, 0x70, 0xeb, 0x73, 0x3a, 0x58, 0x99, 0xa6,
0xaf, 0xc1, 0x39, 0x4b, 0x2f, 0xb1, 0x2f, 0xb8, 0x50, 0xf7, 0x39, 0x4a, 0x0b, 0x51, 0x69, 0x91,
0x67, 0xb5, 0x0f, 0x32, 0x59, 0x59, 0xa0, 0x2b, 0x3d, 0xb0, 0x02, 0xbf, 0x70, 0x2d, 0x45, 0x7e,
0xb7, 0x2d, 0x90, 0xc9, 0x84, 0xdf, 0xfd, 0x36, 0xf3, 0x8c, 0xef, 0x33, 0xcf, 0xf8, 0x31, 0xf3,
0x8c, 0xcf, 0x3f, 0xbd, 0x3b, 0x83, 0x65, 0xf5, 0x0b, 0x7d, 0xfa, 0x2b, 0x00, 0x00, 0xff, 0xff,
0x3a, 0x23, 0x0f, 0xb4, 0x53, 0x05, 0x00, 0x00,
// 596 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x94, 0x54, 0xdd, 0x4e, 0x13, 0x41,
0x14, 0x76, 0x7f, 0x68, 0xe8, 0x41, 0x4a, 0x19, 0x8d, 0x59, 0x89, 0x69, 0xea, 0xc4, 0x08, 0xf1,
0x82, 0x0b, 0xbc, 0x31, 0xc4, 0xab, 0x65, 0x51, 0x9a, 0x00, 0x89, 0x03, 0x7a, 0x3f, 0x94, 0x13,
0xdd, 0xb0, 0xdd, 0xad, 0xbb, 0x53, 0xa0, 0xde, 0xfa, 0x12, 0x26, 0x3e, 0x83, 0xef, 0xe1, 0xa5,
0x8f, 0x60, 0xea, 0x8b, 0x98, 0x39, 0x33, 0xfb, 0x63, 0x29, 0x51, 0xef, 0xe6, 0x7c, 0xe7, 0xef,
0x9b, 0x6f, 0xbf, 0x59, 0x58, 0x1d, 0xe7, 0xf1, 0xa5, 0x54, 0xb8, 0x3d, 0xce, 0x33, 0x95, 0xb1,
0xe5, 0x38, 0x55, 0x98, 0xa7, 0x32, 0xe1, 0x87, 0xd0, 0x8a, 0xc2, 0x23, 0x54, 0x92, 0xf5, 0x61,
0x65, 0x2f, 0x4b, 0x26, 0xa3, 0xf4, 0x50, 0x9e, 0x61, 0x12, 0x38, 0x7d, 0x67, 0xab, 0x2d, 0x9a,
0x90, 0xae, 0x38, 0x8d, 0x47, 0xf8, 0x66, 0x22, 0x53, 0x35, 0x19, 0x05, 0xae, 0xa9, 0x68, 0x40,
0xfc, 0x9b, 0x03, 0xed, 0x57, 0xb9, 0x1c, 0x21, 0x4d, 0xdc, 0x80, 0x65, 0x91, 0x5d, 0x35, 0xc7,
0x55, 0x31, 0x7b, 0x0a, 0x9d, 0x41, 0x7a, 0x89, 0x79, 0x81, 0xfb, 0xa9, 0x3c, 0x4b, 0xf0, 0x9c,
0xc6, 0x2d, 0x8b, 0x39, 0x94, 0x3d, 0x82, 0xf6, 0x9e, 0x1c, 0x7e, 0xc0, 0xd3, 0xe9, 0x18, 0x03,
0x8f, 0x86, 0xd4, 0x40, 0x95, 0x3d, 0x89, 0x3f, 0x61, 0xe0, 0xf7, 0x9d, 0xad, 0x55, 0x51, 0x03,
0xf3, 0x7c, 0x97, 0x6e, 0xf2, 0xe5, 0xd0, 0x19, 0x8c, 0xc6, 0x59, 0xae, 0x04, 0x16, 0xe3, 0x2c,
0x2d, 0x90, 0x75, 0xc1, 0xdb, 0xcf, 0x73, 0x4b, 0x57, 0x1f, 0xf9, 0x35, 0x74, 0xc3, 0x24, 0x1b,
0x5e, 0x44, 0x52, 0x49, 0x81, 0x1f, 0x27, 0x58, 0x28, 0xd6, 0x01, 0x37, 0x0a, 0x6d, 0x91, 0x1b,
0x85, 0xec, 0x3e, 0x2c, 0xd1, 0xb5, 0xad, 0x26, 0x26, 0xd0, 0x28, 0x75, 0x12, 0x6f, 0x5f, 0x98,
0x40, 0xa3, 0x27, 0x49, 0x3c, 0x34, 0x7c, 0x7d, 0x61, 0x02, 0xc6, 0xc0, 0x7f, 0x17, 0xe3, 0x95,
0x25, 0x49, 0x67, 0x3e, 0x80, 0xf5, 0xc6, 0x66, 0x4b, 0xf0, 0x01, 0xb4, 0x44, 0x76, 0x35, 0x88,
0x8a, 0xc0, 0xe9, 0x7b, 0x5b, 0xbe, 0xb0, 0x11, 0x49, 0x41, 0xdf, 0x4a, 0xa7, 0x5c, 0x4a, 0xd5,
0x00, 0x7f, 0x08, 0x4b, 0xa4, 0x8b, 0xbe, 0x5f, 0xdd, 0xab, 0x8f, 0xfc, 0xab, 0x03, 0xeb, 0x47,
0xf2, 0x9a, 0x68, 0x14, 0xd5, 0x9a, 0x03, 0x68, 0x57, 0x20, 0x55, 0xaf, 0xec, 0x3c, 0xdb, 0x2e,
0x5d, 0xb3, 0x7d, 0xa3, 0xbe, 0x46, 0xf6, 0x53, 0x95, 0x4f, 0x45, 0xdd, 0xbc, 0xf1, 0x12, 0x3a,
0x7f, 0x26, 0x35, 0x87, 0x0b, 0x9c, 0x96, 0x1a, 0x5f, 0xe0, 0x54, 0x6b, 0x72, 0x29, 0x93, 0x89,
0xd1, 0xcf, 0x17, 0x26, 0xd8, 0x75, 0x5f, 0x38, 0x7c, 0x17, 0xd8, 0x5e, 0x8e, 0x52, 0x21, 0x0d,
0x38, 0xc2, 0xa2, 0x90, 0xef, 0x71, 0x91, 0xfe, 0x46, 0x53, 0xb7, 0xa1, 0x29, 0x7f, 0x0c, 0x6b,
0x11, 0x26, 0xa8, 0x50, 0x3b, 0x7c, 0x61, 0x23, 0x7f, 0x0d, 0x6b, 0x66, 0xfc, 0xad, 0x25, 0xec,
0x09, 0xf8, 0xda, 0xcd, 0x34, 0x7a, 0x65, 0xa7, 0x5b, 0x8b, 0x60, 0xde, 0x8d, 0xa0, 0x2c, 0x1f,
0x96, 0x3c, 0xad, 0xfd, 0x6f, 0xe5, 0xb9, 0xc0, 0x27, 0x9b, 0x76, 0x83, 0x47, 0x1b, 0xee, 0xd5,
0x1b, 0xaa, 0xa7, 0x64, 0x97, 0xec, 0x02, 0x33, 0x17, 0xfa, 0xff, 0x25, 0x3c, 0xb2, 0xa8, 0x76,
0xda, 0xb1, 0xce, 0x9a, 0x06, 0x3a, 0x57, 0x0c, 0xdc, 0xbf, 0x31, 0xf8, 0xec, 0xe8, 0x65, 0x0b,
0x67, 0xfc, 0x93, 0x4e, 0xfa, 0x9f, 0x50, 0xba, 0xc1, 0x3e, 0x8b, 0x2a, 0x66, 0x9b, 0xd0, 0xa2,
0x7d, 0x45, 0xe0, 0x93, 0xe1, 0xd6, 0xe6, 0x78, 0x08, 0x9b, 0xe6, 0x6f, 0xa1, 0x7d, 0x9c, 0x9d,
0xe3, 0x89, 0x92, 0x8a, 0xee, 0x73, 0x90, 0x15, 0xaa, 0xe4, 0xa2, 0xcf, 0xe4, 0x07, 0x9d, 0x2c,
0x25, 0x30, 0x95, 0x3d, 0xf0, 0xa2, 0xb0, 0x08, 0x3c, 0x1a, 0x7e, 0xb7, 0x49, 0x50, 0xe8, 0x44,
0xd8, 0xfd, 0x3e, 0xeb, 0x39, 0x3f, 0x66, 0x3d, 0xe7, 0xe7, 0xac, 0xe7, 0x7c, 0xf9, 0xd5, 0xbb,
0x73, 0xd6, 0xa2, 0xdf, 0xe5, 0xf3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe6, 0xcf, 0x20, 0xf1,
0x3f, 0x05, 0x00, 0x00,
}

6
internal/private.proto
View file

@ -28,12 +28,12 @@ message BlockDataRequest {
}
message BlockDataResponse {
repeated uint64 BitmapIDs = 1;
repeated uint64 ProfileIDs = 2;
repeated uint64 RowIDs = 1;
repeated uint64 ColumnIDs = 2;
}
message Cache {
repeated uint64 BitmapIDs = 1;
repeated uint64 IDs = 1;
}
message MaxSlicesResponse {

215
internal/public.pb.go
View file

@ -12,7 +12,7 @@
Bitmap
Pair
Bit
Profile
ColumnAttrSet
Attr
AttrMap
QueryRequest
@ -67,8 +67,8 @@ func (*Pair) ProtoMessage() {}
func (*Pair) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{1} }
type Bit struct {
BitmapID uint64 `protobuf:"varint,1,opt,name=BitmapID,proto3" json:"BitmapID,omitempty"`
ProfileID uint64 `protobuf:"varint,2,opt,name=ProfileID,proto3" json:"ProfileID,omitempty"`
RowID uint64 `protobuf:"varint,1,opt,name=RowID,proto3" json:"RowID,omitempty"`
ColumnID uint64 `protobuf:"varint,2,opt,name=ColumnID,proto3" json:"ColumnID,omitempty"`
Timestamp int64 `protobuf:"varint,3,opt,name=Timestamp,proto3" json:"Timestamp,omitempty"`
}
@ -77,17 +77,17 @@ func (m *Bit) String() string { return proto.CompactTextString(m) }
func (*Bit) ProtoMessage() {}
func (*Bit) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{2} }
type Profile struct {
type ColumnAttrSet struct {
ID uint64 `protobuf:"varint,1,opt,name=ID,proto3" json:"ID,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
}
func (m *Profile) Reset() { *m = Profile{} }
func (m *Profile) String() string { return proto.CompactTextString(m) }
func (*Profile) ProtoMessage() {}
func (*Profile) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} }
func (m *ColumnAttrSet) Reset() { *m = ColumnAttrSet{} }
func (m *ColumnAttrSet) String() string { return proto.CompactTextString(m) }
func (*ColumnAttrSet) ProtoMessage() {}
func (*ColumnAttrSet) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{3} }
func (m *Profile) GetAttrs() []*Attr {
func (m *ColumnAttrSet) GetAttrs() []*Attr {
if m != nil {
return m.Attrs
}
@ -125,11 +125,11 @@ func (m *AttrMap) GetAttrs() []*Attr {
}
type QueryRequest struct {
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
Profiles bool `protobuf:"varint,3,opt,name=Profiles,proto3" json:"Profiles,omitempty"`
Quantum string `protobuf:"bytes,4,opt,name=Quantum,proto3" json:"Quantum,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
Quantum string `protobuf:"bytes,4,opt,name=Quantum,proto3" json:"Quantum,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
}
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
@ -138,9 +138,9 @@ func (*QueryRequest) ProtoMessage() {}
func (*QueryRequest) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{6} }
type QueryResponse struct {
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"`
Profiles []*Profile `protobuf:"bytes,3,rep,name=Profiles" json:"Profiles,omitempty"`
Err string `protobuf:"bytes,1,opt,name=Err,proto3" json:"Err,omitempty"`
Results []*QueryResult `protobuf:"bytes,2,rep,name=Results" json:"Results,omitempty"`
ColumnAttrSets []*ColumnAttrSet `protobuf:"bytes,3,rep,name=ColumnAttrSets" json:"ColumnAttrSets,omitempty"`
}
func (m *QueryResponse) Reset() { *m = QueryResponse{} }
@ -155,9 +155,9 @@ func (m *QueryResponse) GetResults() []*QueryResult {
return nil
}
func (m *QueryResponse) GetProfiles() []*Profile {
func (m *QueryResponse) GetColumnAttrSets() []*ColumnAttrSet {
if m != nil {
return m.Profiles
return m.ColumnAttrSets
}
return nil
}
@ -192,8 +192,8 @@ type ImportRequest struct {
DB string `protobuf:"bytes,1,opt,name=DB,proto3" json:"DB,omitempty"`
Frame string `protobuf:"bytes,2,opt,name=Frame,proto3" json:"Frame,omitempty"`
Slice uint64 `protobuf:"varint,3,opt,name=Slice,proto3" json:"Slice,omitempty"`
BitmapIDs []uint64 `protobuf:"varint,4,rep,packed,name=BitmapIDs" json:"BitmapIDs,omitempty"`
ProfileIDs []uint64 `protobuf:"varint,5,rep,packed,name=ProfileIDs" json:"ProfileIDs,omitempty"`
RowIDs []uint64 `protobuf:"varint,4,rep,packed,name=RowIDs" json:"RowIDs,omitempty"`
ColumnIDs []uint64 `protobuf:"varint,5,rep,packed,name=ColumnIDs" json:"ColumnIDs,omitempty"`
Timestamps []int64 `protobuf:"varint,6,rep,packed,name=Timestamps" json:"Timestamps,omitempty"`
}
@ -206,7 +206,7 @@ func init() {
proto.RegisterType((*Bitmap)(nil), "internal.Bitmap")
proto.RegisterType((*Pair)(nil), "internal.Pair")
proto.RegisterType((*Bit)(nil), "internal.Bit")
proto.RegisterType((*Profile)(nil), "internal.Profile")
proto.RegisterType((*ColumnAttrSet)(nil), "internal.ColumnAttrSet")
proto.RegisterType((*Attr)(nil), "internal.Attr")
proto.RegisterType((*AttrMap)(nil), "internal.AttrMap")
proto.RegisterType((*QueryRequest)(nil), "internal.QueryRequest")
@ -304,15 +304,15 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if m.BitmapID != 0 {
if m.RowID != 0 {
dAtA[i] = 0x8
i++
i = encodeVarintPublic(dAtA, i, uint64(m.BitmapID))
i = encodeVarintPublic(dAtA, i, uint64(m.RowID))
}
if m.ProfileID != 0 {
if m.ColumnID != 0 {
dAtA[i] = 0x10
i++
i = encodeVarintPublic(dAtA, i, uint64(m.ProfileID))
i = encodeVarintPublic(dAtA, i, uint64(m.ColumnID))
}
if m.Timestamp != 0 {
dAtA[i] = 0x18
@ -322,7 +322,7 @@ func (m *Bit) MarshalTo(dAtA []byte) (int, error) {
return i, nil
}
func (m *Profile) Marshal() (dAtA []byte, err error) {
func (m *ColumnAttrSet) Marshal() (dAtA []byte, err error) {
size := m.Size()
dAtA = make([]byte, size)
n, err := m.MarshalTo(dAtA)
@ -332,7 +332,7 @@ func (m *Profile) Marshal() (dAtA []byte, err error) {
return dAtA[:n], nil
}
func (m *Profile) MarshalTo(dAtA []byte) (int, error) {
func (m *ColumnAttrSet) MarshalTo(dAtA []byte) (int, error) {
var i int
_ = i
var l int
@ -480,10 +480,10 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(j3))
i += copy(dAtA[i:], dAtA4[:j3])
}
if m.Profiles {
if m.ColumnAttrs {
dAtA[i] = 0x18
i++
if m.Profiles {
if m.ColumnAttrs {
dAtA[i] = 1
} else {
dAtA[i] = 0
@ -542,8 +542,8 @@ func (m *QueryResponse) MarshalTo(dAtA []byte) (int, error) {
i += n
}
}
if len(m.Profiles) > 0 {
for _, msg := range m.Profiles {
if len(m.ColumnAttrSets) > 0 {
for _, msg := range m.ColumnAttrSets {
dAtA[i] = 0x1a
i++
i = encodeVarintPublic(dAtA, i, uint64(msg.Size()))
@ -644,10 +644,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintPublic(dAtA, i, uint64(m.Slice))
}
if len(m.BitmapIDs) > 0 {
dAtA7 := make([]byte, len(m.BitmapIDs)*10)
if len(m.RowIDs) > 0 {
dAtA7 := make([]byte, len(m.RowIDs)*10)
var j6 int
for _, num := range m.BitmapIDs {
for _, num := range m.RowIDs {
for num >= 1<<7 {
dAtA7[j6] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -661,10 +661,10 @@ func (m *ImportRequest) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPublic(dAtA, i, uint64(j6))
i += copy(dAtA[i:], dAtA7[:j6])
}
if len(m.ProfileIDs) > 0 {
dAtA9 := make([]byte, len(m.ProfileIDs)*10)
if len(m.ColumnIDs) > 0 {
dAtA9 := make([]byte, len(m.ColumnIDs)*10)
var j8 int
for _, num := range m.ProfileIDs {
for _, num := range m.ColumnIDs {
for num >= 1<<7 {
dAtA9[j8] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -760,11 +760,11 @@ func (m *Pair) Size() (n int) {
func (m *Bit) Size() (n int) {
var l int
_ = l
if m.BitmapID != 0 {
n += 1 + sovPublic(uint64(m.BitmapID))
if m.RowID != 0 {
n += 1 + sovPublic(uint64(m.RowID))
}
if m.ProfileID != 0 {
n += 1 + sovPublic(uint64(m.ProfileID))
if m.ColumnID != 0 {
n += 1 + sovPublic(uint64(m.ColumnID))
}
if m.Timestamp != 0 {
n += 1 + sovPublic(uint64(m.Timestamp))
@ -772,7 +772,7 @@ func (m *Bit) Size() (n int) {
return n
}
func (m *Profile) Size() (n int) {
func (m *ColumnAttrSet) Size() (n int) {
var l int
_ = l
if m.ID != 0 {
@ -839,7 +839,7 @@ func (m *QueryRequest) Size() (n int) {
}
n += 1 + sovPublic(uint64(l)) + l
}
if m.Profiles {
if m.ColumnAttrs {
n += 2
}
l = len(m.Quantum)
@ -865,8 +865,8 @@ func (m *QueryResponse) Size() (n int) {
n += 1 + l + sovPublic(uint64(l))
}
}
if len(m.Profiles) > 0 {
for _, e := range m.Profiles {
if len(m.ColumnAttrSets) > 0 {
for _, e := range m.ColumnAttrSets {
l = e.Size()
n += 1 + l + sovPublic(uint64(l))
}
@ -910,16 +910,16 @@ func (m *ImportRequest) Size() (n int) {
if m.Slice != 0 {
n += 1 + sovPublic(uint64(m.Slice))
}
if len(m.BitmapIDs) > 0 {
if len(m.RowIDs) > 0 {
l = 0
for _, e := range m.BitmapIDs {
for _, e := range m.RowIDs {
l += sovPublic(uint64(e))
}
n += 1 + sovPublic(uint64(l)) + l
}
if len(m.ProfileIDs) > 0 {
if len(m.ColumnIDs) > 0 {
l = 0
for _, e := range m.ProfileIDs {
for _, e := range m.ColumnIDs {
l += sovPublic(uint64(e))
}
n += 1 + sovPublic(uint64(l)) + l
@ -1209,9 +1209,9 @@ func (m *Bit) Unmarshal(dAtA []byte) error {
switch fieldNum {
case 1:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field BitmapID", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field RowID", wireType)
}
m.BitmapID = 0
m.RowID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -1221,16 +1221,16 @@ func (m *Bit) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.BitmapID |= (uint64(b) & 0x7F) << shift
m.RowID |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ProfileID", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ColumnID", wireType)
}
m.ProfileID = 0
m.ColumnID = 0
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPublic
@ -1240,7 +1240,7 @@ func (m *Bit) Unmarshal(dAtA []byte) error {
}
b := dAtA[iNdEx]
iNdEx++
m.ProfileID |= (uint64(b) & 0x7F) << shift
m.ColumnID |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
@ -1285,7 +1285,7 @@ func (m *Bit) Unmarshal(dAtA []byte) error {
}
return nil
}
func (m *Profile) Unmarshal(dAtA []byte) error {
func (m *ColumnAttrSet) Unmarshal(dAtA []byte) error {
l := len(dAtA)
iNdEx := 0
for iNdEx < l {
@ -1308,10 +1308,10 @@ func (m *Profile) Unmarshal(dAtA []byte) error {
fieldNum := int32(wire >> 3)
wireType := int(wire & 0x7)
if wireType == 4 {
return fmt.Errorf("proto: Profile: wiretype end group for non-group")
return fmt.Errorf("proto: ColumnAttrSet: wiretype end group for non-group")
}
if fieldNum <= 0 {
return fmt.Errorf("proto: Profile: illegal tag %d (wire type %d)", fieldNum, wire)
return fmt.Errorf("proto: ColumnAttrSet: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 1:
@ -1772,7 +1772,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ColumnAttrs", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
@ -1789,7 +1789,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.Profiles = bool(v != 0)
m.ColumnAttrs = bool(v != 0)
case 4:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Quantum", wireType)
@ -1951,7 +1951,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error {
iNdEx = postIndex
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Profiles", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ColumnAttrSets", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
@ -1975,8 +1975,8 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error {
if postIndex > l {
return io.ErrUnexpectedEOF
}
m.Profiles = append(m.Profiles, &Profile{})
if err := m.Profiles[len(m.Profiles)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
m.ColumnAttrSets = append(m.ColumnAttrSets, &ColumnAttrSet{})
if err := m.ColumnAttrSets[len(m.ColumnAttrSets)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil {
return err
}
iNdEx = postIndex
@ -2300,7 +2300,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.BitmapIDs = append(m.BitmapIDs, v)
m.RowIDs = append(m.RowIDs, v)
}
} else if wireType == 0 {
var v uint64
@ -2318,9 +2318,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.BitmapIDs = append(m.BitmapIDs, v)
m.RowIDs = append(m.RowIDs, v)
} else {
return fmt.Errorf("proto: wrong wireType = %d for field BitmapIDs", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field RowIDs", wireType)
}
case 5:
if wireType == 2 {
@ -2362,7 +2362,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.ProfileIDs = append(m.ProfileIDs, v)
m.ColumnIDs = append(m.ColumnIDs, v)
}
} else if wireType == 0 {
var v uint64
@ -2380,9 +2380,9 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.ProfileIDs = append(m.ProfileIDs, v)
m.ColumnIDs = append(m.ColumnIDs, v)
} else {
return fmt.Errorf("proto: wrong wireType = %d for field ProfileIDs", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ColumnIDs", wireType)
}
case 6:
if wireType == 2 {
@ -2575,41 +2575,42 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 570 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x54, 0x4b, 0x6e, 0xd4, 0x40,
0x10, 0xa5, 0x6d, 0xcf, 0xaf, 0x26, 0x19, 0x0d, 0x2d, 0x40, 0x16, 0x42, 0x23, 0xcb, 0x62, 0xe1,
0x0d, 0x13, 0x29, 0x1c, 0x00, 0xe1, 0x4c, 0x22, 0x8d, 0x10, 0x51, 0xd2, 0x89, 0xd8, 0xb1, 0x70,
0x42, 0x13, 0x2c, 0xf9, 0x47, 0x77, 0x7b, 0x31, 0x4b, 0x16, 0x6c, 0x38, 0x01, 0x47, 0x80, 0x9b,
0xb0, 0xe4, 0x08, 0x68, 0xb8, 0x08, 0xaa, 0xfe, 0xd8, 0x66, 0x83, 0xd8, 0xf5, 0x7b, 0xe5, 0xea,
0xae, 0xf7, 0xaa, 0xca, 0x70, 0xd0, 0xb4, 0x37, 0x45, 0x7e, 0xbb, 0x6e, 0x44, 0xad, 0x6a, 0x3a,
0xcd, 0x2b, 0xc5, 0x45, 0x95, 0x15, 0x71, 0x0a, 0xe3, 0x34, 0x57, 0x65, 0xd6, 0x50, 0x0a, 0x41,
0x9a, 0x2b, 0x19, 0x92, 0xc8, 0x4f, 0x02, 0xa6, 0xcf, 0xf4, 0x29, 0x8c, 0x5e, 0x2a, 0x25, 0x64,
0xe8, 0x45, 0x7e, 0x32, 0x3f, 0x5e, 0xac, 0x5d, 0xde, 0x1a, 0x69, 0x66, 0x82, 0xf1, 0x1a, 0x82,
0x8b, 0x2c, 0x17, 0x74, 0x09, 0xfe, 0x2b, 0xbe, 0x0b, 0x49, 0x44, 0x92, 0x80, 0xe1, 0x91, 0x3e,
0x80, 0xd1, 0x49, 0xdd, 0x56, 0x2a, 0xf4, 0x34, 0x67, 0x40, 0xfc, 0x16, 0xfc, 0x34, 0x57, 0xf4,
0x31, 0x4c, 0xcd, 0xd3, 0xdb, 0x8d, 0xcd, 0xe9, 0x30, 0x7d, 0x02, 0xb3, 0x0b, 0x51, 0xbf, 0xcf,
0x0b, 0xbe, 0xdd, 0xd8, 0xe4, 0x9e, 0xc0, 0xe8, 0x75, 0x5e, 0x72, 0xa9, 0xb2, 0xb2, 0x09, 0xfd,
0x88, 0x24, 0x3e, 0xeb, 0x89, 0xf8, 0x05, 0x4c, 0xec, 0xa7, 0x74, 0x01, 0x5e, 0x77, 0xb9, 0xb7,
0xdd, 0xfc, 0xa7, 0x9e, 0x6f, 0x04, 0x02, 0x3c, 0x0d, 0x05, 0xcd, 0x8c, 0x20, 0x0a, 0xc1, 0xf5,
0xae, 0xe1, 0xb6, 0x24, 0x7d, 0xa6, 0x11, 0xcc, 0xaf, 0x94, 0xc8, 0xab, 0xbb, 0x37, 0x59, 0xd1,
0x72, 0x5d, 0xcf, 0x8c, 0x0d, 0x29, 0x54, 0xba, 0xad, 0x94, 0x09, 0x07, 0xba, 0xdc, 0x0e, 0xa3,
0x96, 0xb4, 0xae, 0x0b, 0x13, 0x1c, 0x45, 0x24, 0x99, 0xb2, 0x9e, 0xa0, 0x2b, 0x80, 0xb3, 0xa2,
0xce, 0x6c, 0xee, 0x38, 0x22, 0x09, 0x61, 0x03, 0x26, 0x3e, 0x82, 0x09, 0x56, 0xfa, 0x3a, 0x6b,
0x7a, 0x6d, 0xe4, 0x5f, 0xda, 0xbe, 0x10, 0x38, 0xb8, 0x6c, 0xb9, 0xd8, 0x31, 0xfe, 0xb1, 0xe5,
0x52, 0x61, 0x8b, 0x34, 0xb6, 0x2a, 0x0d, 0xa0, 0x8f, 0x60, 0x7c, 0x55, 0xe4, 0xb7, 0xdc, 0x38,
0x15, 0x30, 0x8b, 0x50, 0x89, 0xf5, 0x56, 0x6a, 0xa1, 0x53, 0xd6, 0x61, 0x1a, 0xc2, 0xe4, 0xb2,
0xcd, 0x2a, 0xd5, 0x96, 0x5a, 0xe4, 0x8c, 0x39, 0x88, 0xb7, 0x31, 0x5e, 0xd6, 0xca, 0x09, 0xb4,
0x28, 0xfe, 0x44, 0xe0, 0xd0, 0x16, 0x23, 0x9b, 0xba, 0x92, 0x1c, 0x1d, 0x3f, 0x15, 0xc2, 0x39,
0x7e, 0x2a, 0x04, 0x3d, 0x82, 0x09, 0xe3, 0xb2, 0x2d, 0x94, 0x6b, 0xda, 0xc3, 0x5e, 0x98, 0xcb,
0x6d, 0x0b, 0xc5, 0xdc, 0x57, 0xf4, 0xd9, 0x5f, 0x25, 0x62, 0xc6, 0xfd, 0x3e, 0xc3, 0x46, 0xfa,
0xaa, 0xe3, 0xcf, 0x04, 0xe6, 0x83, 0x7b, 0x68, 0xe2, 0x16, 0x42, 0x17, 0x31, 0x3f, 0x5e, 0xf6,
0xc9, 0x86, 0x67, 0x6e, 0x61, 0x0e, 0x80, 0x9c, 0xdb, 0x41, 0x20, 0xe7, 0x68, 0x3f, 0x2e, 0x81,
0x7b, 0x73, 0x60, 0x3f, 0xd2, 0xcc, 0x04, 0xd1, 0xa3, 0x93, 0x0f, 0x59, 0x75, 0xc7, 0xdf, 0x69,
0x8f, 0xa6, 0xcc, 0xc1, 0xf8, 0x3b, 0x81, 0xc3, 0x6d, 0xd9, 0xd4, 0x42, 0xb9, 0xce, 0x2c, 0xc0,
0xdb, 0xa4, 0xd6, 0x0a, 0x6f, 0x93, 0x62, 0xa7, 0xce, 0x44, 0x56, 0x9a, 0xe1, 0x9b, 0x31, 0x03,
0x90, 0xd5, 0xbd, 0xd1, 0xed, 0x08, 0x98, 0x01, 0x7a, 0xaa, 0xec, 0x2e, 0xc9, 0x30, 0xd0, 0x2d,
0xec, 0x09, 0x9c, 0xaa, 0x6e, 0x99, 0x64, 0x38, 0xd2, 0xe1, 0x01, 0x83, 0xf1, 0x6e, 0x9d, 0x64,
0x38, 0x8e, 0xfc, 0xc4, 0x67, 0x03, 0x26, 0x5d, 0xfe, 0xd8, 0xaf, 0xc8, 0xcf, 0xfd, 0x8a, 0xfc,
0xda, 0xaf, 0xc8, 0xd7, 0xdf, 0xab, 0x7b, 0x37, 0x63, 0xfd, 0x5f, 0x79, 0xfe, 0x27, 0x00, 0x00,
0xff, 0xff, 0x37, 0xb6, 0x15, 0x22, 0x67, 0x04, 0x00, 0x00,
// 579 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x54, 0xcd, 0x6e, 0xd3, 0x40,
0x10, 0x66, 0x6d, 0x27, 0x4d, 0x26, 0x6d, 0x14, 0xad, 0xf8, 0xb1, 0x10, 0x8a, 0x2c, 0x8b, 0x83,
0x4f, 0xa9, 0x54, 0x1e, 0x00, 0xe1, 0x24, 0x95, 0x22, 0x44, 0x45, 0x27, 0x85, 0xbb, 0x5b, 0x56,
0xc5, 0x92, 0xff, 0x58, 0xaf, 0x85, 0xf2, 0x00, 0xdc, 0x91, 0xb8, 0x70, 0xe5, 0xc6, 0xa3, 0x70,
0xe4, 0x11, 0x50, 0x78, 0x11, 0x34, 0xbb, 0xde, 0xd8, 0xe5, 0x80, 0xb8, 0xed, 0xf7, 0xcd, 0xce,
0x7a, 0xbe, 0xf9, 0x66, 0x0c, 0xc7, 0x55, 0x73, 0x9d, 0xa5, 0x37, 0x8b, 0x4a, 0x96, 0xaa, 0xe4,
0xa3, 0xb4, 0x50, 0x42, 0x16, 0x49, 0x16, 0xc6, 0x30, 0x8c, 0x53, 0x95, 0x27, 0x15, 0xe7, 0xe0,
0xc5, 0xa9, 0xaa, 0x7d, 0x16, 0xb8, 0x91, 0x87, 0xfa, 0xcc, 0x9f, 0xc2, 0xe0, 0x85, 0x52, 0xb2,
0xf6, 0x9d, 0xc0, 0x8d, 0x26, 0x67, 0xd3, 0x85, 0xcd, 0x5b, 0x10, 0x8d, 0x26, 0x18, 0x2e, 0xc0,
0x7b, 0x9d, 0xa4, 0x92, 0xcf, 0xc0, 0x7d, 0x29, 0x76, 0x3e, 0x0b, 0x58, 0xe4, 0x21, 0x1d, 0xf9,
0x7d, 0x18, 0x2c, 0xcb, 0xa6, 0x50, 0xbe, 0xa3, 0x39, 0x03, 0xc2, 0x37, 0xe0, 0xc6, 0xa9, 0xa2,
0x20, 0x96, 0x1f, 0x37, 0xab, 0x36, 0xc1, 0x00, 0xfe, 0x18, 0x46, 0xcb, 0x32, 0x6b, 0xf2, 0x62,
0xb3, 0x6a, 0xb3, 0x0e, 0x98, 0x3f, 0x81, 0xf1, 0x55, 0x9a, 0x8b, 0x5a, 0x25, 0x79, 0xe5, 0xbb,
0x01, 0x8b, 0x5c, 0xec, 0x88, 0x70, 0x0d, 0x27, 0xe6, 0x26, 0x55, 0xb5, 0x15, 0x8a, 0x4f, 0xc1,
0x39, 0xbc, 0xee, 0x6c, 0x56, 0xff, 0xa9, 0xe6, 0x3b, 0x03, 0x8f, 0x4e, 0x7d, 0x39, 0x63, 0x23,
0x87, 0x83, 0x77, 0xb5, 0xab, 0x44, 0x5b, 0x97, 0x3e, 0xf3, 0x00, 0x26, 0x5b, 0x25, 0xd3, 0xe2,
0xf6, 0x6d, 0x92, 0x35, 0x42, 0x57, 0x35, 0xc6, 0x3e, 0x45, 0x8a, 0x36, 0x85, 0x32, 0x61, 0x4f,
0x17, 0x7d, 0xc0, 0xa4, 0x28, 0x2e, 0xcb, 0xcc, 0x04, 0x07, 0x01, 0x8b, 0x46, 0xd8, 0x11, 0x7c,
0x0e, 0x70, 0x9e, 0x95, 0x49, 0x9b, 0x3b, 0x0c, 0x58, 0xc4, 0xb0, 0xc7, 0x84, 0xa7, 0x70, 0x44,
0x95, 0xbe, 0x4a, 0xaa, 0x4e, 0x1b, 0xfb, 0x97, 0xb6, 0xcf, 0x0c, 0x8e, 0x2f, 0x1b, 0x21, 0x77,
0x28, 0x3e, 0x34, 0xa2, 0xd6, 0x1e, 0x68, 0xdc, 0xaa, 0x34, 0x80, 0x3f, 0x84, 0xe1, 0x36, 0x4b,
0x6f, 0x84, 0xe9, 0x94, 0x87, 0x2d, 0x22, 0xad, 0x5d, 0x87, 0x6b, 0xad, 0x75, 0x84, 0x7d, 0x8a,
0xfb, 0x70, 0x74, 0xd9, 0x24, 0x85, 0x6a, 0x72, 0x2d, 0x75, 0x8c, 0x16, 0xd2, 0x9b, 0x28, 0xf2,
0x52, 0x59, 0x99, 0x2d, 0x0a, 0xbf, 0x30, 0x38, 0x69, 0x4b, 0xaa, 0xab, 0xb2, 0xa8, 0x05, 0xf5,
0x7d, 0x2d, 0xa5, 0xed, 0xfb, 0x5a, 0x4a, 0x7e, 0x0a, 0x47, 0x28, 0xea, 0x26, 0x53, 0xd6, 0xba,
0x07, 0x9d, 0x3c, 0x9b, 0xdb, 0x64, 0x0a, 0xed, 0x2d, 0xfe, 0x1c, 0xa6, 0x77, 0x46, 0x81, 0x6a,
0xa5, 0xbc, 0x47, 0x5d, 0xde, 0x9d, 0x38, 0xfe, 0x75, 0x3d, 0xfc, 0xc4, 0x60, 0xd2, 0x7b, 0x99,
0x47, 0x76, 0x4d, 0x74, 0x59, 0x93, 0xb3, 0x59, 0xf7, 0x90, 0xe1, 0xd1, 0xae, 0xd1, 0x31, 0xb0,
0x8b, 0x76, 0x40, 0xd8, 0x05, 0xd9, 0x42, 0xab, 0x61, 0xbf, 0xdf, 0xb3, 0x85, 0x68, 0x34, 0x41,
0xea, 0xda, 0xf2, 0x7d, 0x52, 0xdc, 0x8a, 0x77, 0xba, 0x6b, 0x23, 0xb4, 0x30, 0xfc, 0xc6, 0xe0,
0x64, 0x93, 0x57, 0xa5, 0x54, 0xd6, 0xb1, 0x29, 0x38, 0xab, 0xb8, 0x6d, 0x8e, 0xb3, 0x8a, 0xc9,
0xc1, 0x73, 0x99, 0xe4, 0x66, 0x28, 0xc7, 0x68, 0x00, 0xb1, 0xda, 0x33, 0xed, 0x91, 0x87, 0x06,
0x68, 0x0f, 0x68, 0xc9, 0x6a, 0xdf, 0x33, 0xbe, 0x1a, 0x44, 0x53, 0x68, 0x77, 0xac, 0xf6, 0x07,
0x3a, 0xd4, 0x11, 0x34, 0x85, 0x87, 0x25, 0xab, 0xfd, 0x61, 0xe0, 0x46, 0x2e, 0xf6, 0x98, 0x78,
0xf6, 0x63, 0x3f, 0x67, 0x3f, 0xf7, 0x73, 0xf6, 0x6b, 0x3f, 0x67, 0x5f, 0x7f, 0xcf, 0xef, 0x5d,
0x0f, 0xf5, 0x5f, 0xe6, 0xd9, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xaf, 0x79, 0x70, 0xf4, 0x75,
0x04, 0x00, 0x00,
}

14
internal/public.proto
View file

@ -13,12 +13,12 @@ message Pair {
}
message Bit {
uint64 BitmapID = 1;
uint64 ProfileID = 2;
uint64 RowID = 1;
uint64 ColumnID = 2;
int64 Timestamp = 3;
}
message Profile {
message ColumnAttrSet {
uint64 ID = 1;
repeated Attr Attrs = 2;
}
@ -39,7 +39,7 @@ message AttrMap {
message QueryRequest {
string Query = 1;
repeated uint64 Slices = 2;
bool Profiles = 3;
bool ColumnAttrs = 3;
string Quantum = 4;
bool Remote = 5;
}
@ -47,7 +47,7 @@ message QueryRequest {
message QueryResponse {
string Err = 1;
repeated QueryResult Results = 2;
repeated Profile Profiles = 3;
repeated ColumnAttrSet ColumnAttrSets = 3;
}
message QueryResult {
@ -61,7 +61,7 @@ message ImportRequest {
string DB = 1;
string Frame = 2;
uint64 Slice = 3;
repeated uint64 BitmapIDs = 4;
repeated uint64 ProfileIDs = 5;
repeated uint64 RowIDs = 4;
repeated uint64 ColumnIDs = 5;
repeated int64 Timestamps = 6;
}

104
iterator.go
View file

@ -6,19 +6,19 @@ import (
"github.com/pilosa/pilosa/roaring"
)
// Iterator is an interface for looping over bitmap/profile pairs.
// Iterator is an interface for looping over row/column pairs.
type Iterator interface {
Seek(bitmapID, profileID uint64)
Next() (bitmapID, profileID uint64, eof bool)
Seek(rowID, columnID uint64)
Next() (rowID, columnID uint64, eof bool)
}
// BufIterator wraps an iterator to provide the ability to unread values.
type BufIterator struct {
buf struct {
bitmapID uint64
profileID uint64
eof bool
full bool
rowID uint64
columnID uint64
eof bool
full bool
}
itr Iterator
}
@ -29,28 +29,28 @@ func NewBufIterator(itr Iterator) *BufIterator {
}
// Seek moves to the first pair equal to or greater than pseek/bseek.
func (itr *BufIterator) Seek(bitmapID, profileID uint64) {
func (itr *BufIterator) Seek(rowID, columnID uint64) {
itr.buf.full = false
itr.itr.Seek(bitmapID, profileID)
itr.itr.Seek(rowID, columnID)
}
// Next returns the next pair in the bitmap.
// Next returns the next pair in the row.
// If a value has been buffered then it is returned and the buffer is cleared.
func (itr *BufIterator) Next() (bitmapID, profileID uint64, eof bool) {
func (itr *BufIterator) Next() (rowID, columnID uint64, eof bool) {
if itr.buf.full {
itr.buf.full = false
return itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof
return itr.buf.rowID, itr.buf.columnID, itr.buf.eof
}
// Read values onto buffer in case of unread.
itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof = itr.itr.Next()
itr.buf.rowID, itr.buf.columnID, itr.buf.eof = itr.itr.Next()
return itr.buf.bitmapID, itr.buf.profileID, itr.buf.eof
return itr.buf.rowID, itr.buf.columnID, itr.buf.eof
}
// Peek reads the next value but leaves it on the buffer.
func (itr *BufIterator) Peek() (bitmapID, profileID uint64, eof bool) {
bitmapID, profileID, eof = itr.Next()
func (itr *BufIterator) Peek() (rowID, columnID uint64, eof bool) {
rowID, columnID, eof = itr.Next()
itr.Unread()
return
}
@ -64,30 +64,30 @@ func (itr *BufIterator) Unread() {
itr.buf.full = true
}
// LimitIterator wraps an Iterator and limits it to a max profile/bitmap pair.
// LimitIterator wraps an Iterator and limits it to a max column/row pair.
type LimitIterator struct {
itr Iterator
maxBitmapID uint64
maxProfileID uint64
itr Iterator
maxRowID uint64
maxColumnID uint64
eof bool
}
// NewLimitIterator returns a new LimitIterator.
func NewLimitIterator(itr Iterator, maxBitmapID, maxProfileID uint64) *LimitIterator {
func NewLimitIterator(itr Iterator, maxRowID, maxColumnID uint64) *LimitIterator {
return &LimitIterator{
itr: itr,
maxBitmapID: maxBitmapID,
maxProfileID: maxProfileID,
itr: itr,
maxRowID: maxRowID,
maxColumnID: maxColumnID,
}
}
// Seek moves the underlying iterator to a profile/bitmap pair.
func (itr *LimitIterator) Seek(bitmapID, profileID uint64) { itr.itr.Seek(bitmapID, profileID) }
// Seek moves the underlying iterator to a column/row pair.
func (itr *LimitIterator) Seek(rowID, columnID uint64) { itr.itr.Seek(rowID, columnID) }
// Next returns the next bitmap/profile ID pair.
// Next returns the next row/column ID pair.
// If the underlying iterator returns a pair higher than the max then EOF is returned.
func (itr *LimitIterator) Next() (bitmapID, profileID uint64, eof bool) {
func (itr *LimitIterator) Next() (rowID, columnID uint64, eof bool) {
// Always return EOF once it is reached by limit or the underlying iterator.
if itr.eof {
return 0, 0, true
@ -95,35 +95,35 @@ func (itr *LimitIterator) Next() (bitmapID, profileID uint64, eof bool) {
// Retrieve pair from underlying iterator.
// Mark as EOF if it is beyond the limit (or at EOF).
bitmapID, profileID, eof = itr.itr.Next()
if eof || bitmapID > itr.maxBitmapID || (bitmapID == itr.maxBitmapID && profileID > itr.maxProfileID) {
rowID, columnID, eof = itr.itr.Next()
if eof || rowID > itr.maxRowID || (rowID == itr.maxRowID && columnID > itr.maxColumnID) {
itr.eof = true
return 0, 0, true
}
return bitmapID, profileID, false
return rowID, columnID, false
}
// SliceIterator iterates over a pair of bitmap/profile ID slices.
// SliceIterator iterates over a pair of row/column ID slices.
type SliceIterator struct {
bitmapIDs []uint64
profileIDs []uint64
rowIDs []uint64
columnIDs []uint64
i, n int
}
// NewSliceIterator returns an iterator to iterate over a set of bitmap/profile ID pairs.
// NewSliceIterator returns an iterator to iterate over a set of row/column ID pairs.
// Both slices MUST have an equal length. Otherwise the function will panic.
func NewSliceIterator(bitmapIDs, profileIDs []uint64) *SliceIterator {
if len(profileIDs) != len(bitmapIDs) {
panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(bitmapIDs), len(profileIDs)))
func NewSliceIterator(rowIDs, columnIDs []uint64) *SliceIterator {
if len(columnIDs) != len(rowIDs) {
panic(fmt.Sprintf("pilosa.SliceIterator: pair length mismatch: %d != %d", len(rowIDs), len(columnIDs)))
}
return &SliceIterator{
bitmapIDs: bitmapIDs,
profileIDs: profileIDs,
rowIDs: rowIDs,
columnIDs: columnIDs,
n: len(bitmapIDs),
n: len(rowIDs),
}
}
@ -131,10 +131,10 @@ func NewSliceIterator(bitmapIDs, profileIDs []uint64) *SliceIterator {
// If the pair is not found, the iterator seeks to the next pair.
func (itr *SliceIterator) Seek(bseek, pseek uint64) {
for i := 0; i < itr.n; i++ {
bitmapID := itr.bitmapIDs[i]
profileID := itr.profileIDs[i]
rowID := itr.rowIDs[i]
columnID := itr.columnIDs[i]
if (bseek == bitmapID && pseek <= profileID) || bseek < bitmapID {
if (bseek == rowID && pseek <= columnID) || bseek < rowID {
itr.i = i
return
}
@ -144,20 +144,20 @@ func (itr *SliceIterator) Seek(bseek, pseek uint64) {
itr.i = itr.n
}
// Next returns the next bitmap/profile ID pair.
func (itr *SliceIterator) Next() (bitmapID, profileID uint64, eof bool) {
// Next returns the next row/column ID pair.
func (itr *SliceIterator) Next() (rowID, columnID uint64, eof bool) {
if itr.i >= itr.n {
return 0, 0, true
}
bitmapID = itr.bitmapIDs[itr.i]
profileID = itr.profileIDs[itr.i]
rowID = itr.rowIDs[itr.i]
columnID = itr.columnIDs[itr.i]
itr.i++
return bitmapID, profileID, false
return rowID, columnID, false
}
// RoaringIterator converts a roaring.Iterator to output profile/bitmap pairs.
// RoaringIterator converts a roaring.Iterator to output column/row pairs.
type RoaringIterator struct {
itr *roaring.Iterator
}
@ -173,8 +173,8 @@ func (itr *RoaringIterator) Seek(bseek, pseek uint64) {
itr.itr.Seek((bseek * SliceWidth) + pseek)
}
// Next returns the next profile/bitmap ID pair.
func (itr *RoaringIterator) Next() (bitmapID, profileID uint64, eof bool) {
// Next returns the next column/row ID pair.
func (itr *RoaringIterator) Next() (rowID, columnID uint64, eof bool) {
v, eof := itr.itr.Next()
return v / SliceWidth, v % SliceWidth, eof
}

44
pilosa.go
View file

@ -35,54 +35,54 @@ var (
// Todo: remove . when frame doesn't require . for topN
var nameRegexp = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,64}$`)
// Profile represents vertical column in a database.
// A profile can have a set of attributes attached to it.
type Profile struct {
// ColumnAttrSet represents a set of attributes for a vertical column in a database.
// Can have a set of attributes attached to it.
type ColumnAttrSet struct {
ID uint64 `json:"id"`
Attrs map[string]interface{} `json:"attrs,omitempty"`
}
// encodeProfiles converts a into its internal representation.
func encodeProfiles(a []*Profile) []*internal.Profile {
other := make([]*internal.Profile, len(a))
// encodeColumnAttrSets converts a into its internal representation.
func encodeColumnAttrSets(a []*ColumnAttrSet) []*internal.ColumnAttrSet {
other := make([]*internal.ColumnAttrSet, len(a))
for i := range a {
other[i] = encodeProfile(a[i])
other[i] = encodeColumnAttrSet(a[i])
}
return other
}
// decodeProfiles converts a from its internal representation.
func decodeProfiles(a []*internal.Profile) []*Profile {
other := make([]*Profile, len(a))
// decodeColumnAttrSets converts a from its internal representation.
func decodeColumnAttrSets(a []*internal.ColumnAttrSet) []*ColumnAttrSet {
other := make([]*ColumnAttrSet, len(a))
for i := range a {
other[i] = decodeProfile(a[i])
other[i] = decodeColumnAttrSet(a[i])
}
return other
}
// encodeProfile converts p into its internal representation.
func encodeProfile(p *Profile) *internal.Profile {
return &internal.Profile{
ID: p.ID,
Attrs: encodeAttrs(p.Attrs),
// encodeColumnAttrSet converts set into its internal representation.
func encodeColumnAttrSet(set *ColumnAttrSet) *internal.ColumnAttrSet {
return &internal.ColumnAttrSet{
ID: set.ID,
Attrs: encodeAttrs(set.Attrs),
}
}
// decodeProfile converts b from its internal representation.
func decodeProfile(pb *internal.Profile) *Profile {
p := &Profile{
// decodeColumnAttrSet converts b from its internal representation.
func decodeColumnAttrSet(pb *internal.ColumnAttrSet) *ColumnAttrSet {
set := &ColumnAttrSet{
ID: pb.ID,
}
if len(pb.Attrs) > 0 {
p.Attrs = make(map[string]interface{}, len(pb.Attrs))
set.Attrs = make(map[string]interface{}, len(pb.Attrs))
for _, attr := range pb.Attrs {
k, v := decodeAttr(attr)
p.Attrs[k] = v
set.Attrs[k] = v
}
}
return p
return set
}
// TimeFormat is the go-style time format used to parse string dates.

126
server/server_test.go
View file

@ -45,18 +45,18 @@ func TestMain_Set_Quick(t *testing.T) {
if err := client.CreateFrame(context.Background(), "d", cmd.Frame, pilosa.FrameOptions{}); err != nil && err != pilosa.ErrFrameExists {
t.Fatal(err)
}
if _, err := m.Query("d", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, profileID=%d)`, cmd.ID, cmd.Frame, cmd.ProfileID)); err != nil {
if _, err := m.Query("d", "", fmt.Sprintf(`SetBit(id=%d, frame=%q, columnID=%d)`, cmd.ID, cmd.Frame, cmd.ColumnID)); err != nil {
t.Fatal(err)
}
}
// Validate data.
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, profileIDs := range frameSet {
for id, columnIDs := range frameSet {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"bits": profileIDs,
"bits": columnIDs,
"attrs": map[string]interface{}{},
},
},
@ -75,11 +75,11 @@ func TestMain_Set_Quick(t *testing.T) {
// Validate data after reopening.
for frame, frameSet := range SetCommands(cmds).Frames() {
for id, profileIDs := range frameSet {
for id, columnIDs := range frameSet {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"bits": profileIDs,
"bits": columnIDs,
"attrs": map[string]interface{}{},
},
},
@ -102,8 +102,8 @@ func TestMain_Set_Quick(t *testing.T) {
}
}
// Ensure program can set bitmap attributes and retrieve them.
func TestMain_SetBitmapAttrs(t *testing.T) {
// Ensure program can set row attributes and retrieve them.
func TestMain_SetRowAttrs(t *testing.T) {
m := MustRunMain()
defer m.Close()
@ -119,36 +119,36 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
t.Fatal(err)
}
// Set bits on different bitmaps in different frames.
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil {
// Set bits on different rows in different frames.
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=2, frame="x.n", profileID=100)`); err != nil {
} else if _, err := m.Query("d", "", `SetBit(id=2, frame="x.n", columnID=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=2, frame="z", profileID=100)`); err != nil {
} else if _, err := m.Query("d", "", `SetBit(id=2, frame="z", columnID=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=3, frame="neg", profileID=100)`); err != nil {
} else if _, err := m.Query("d", "", `SetBit(id=3, frame="neg", columnID=100)`); err != nil {
t.Fatal(err)
}
// Set bitmap attributes.
if _, err := m.Query("d", "", `SetBitmapAttrs(id=1, frame="x.n", x=100)`); err != nil {
// Set row attributes.
if _, err := m.Query("d", "", `SetRowAttrs(id=1, frame="x.n", x=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBitmapAttrs(id=2, frame="x.n", x=-200)`); err != nil {
} else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="x.n", x=-200)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBitmapAttrs(id=2, frame="z", x=300)`); err != nil {
} else if _, err := m.Query("d", "", `SetRowAttrs(id=2, frame="z", x=300)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBitmapAttrs(id=3, frame="neg", x=-0.44)`); err != nil {
} else if _, err := m.Query("d", "", `SetRowAttrs(id=3, frame="neg", x=-0.44)`); err != nil {
t.Fatal(err)
}
// Query bitmap x.n/1.
// Query row x.n/1.
if res, err := m.Query("d", "", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query bitmap x.n/2.
// Query row x.n/2.
if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
@ -159,19 +159,19 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
t.Fatal(err)
}
// Query bitmaps after reopening.
if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
// Query rows after reopening.
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("d", "profiles=true", `Bitmap(id=3, frame="neg")`); err != nil {
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=3, frame="neg")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query bitmap x.n/2.
// Query row x.n/2.
if res, err := m.Query("d", "", `Bitmap(id=2, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
@ -179,8 +179,8 @@ func TestMain_SetBitmapAttrs(t *testing.T) {
}
}
// Ensure program can set profile attributes and retrieve them.
func TestMain_SetProfileAttrs(t *testing.T) {
// Ensure program can set column attributes and retrieve them.
func TestMain_SetColumnAttrs(t *testing.T) {
m := MustRunMain()
defer m.Close()
@ -192,22 +192,22 @@ func TestMain_SetProfileAttrs(t *testing.T) {
t.Fatal(err)
}
// Set bits on bitmap.
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=100)`); err != nil {
// Set bits on row.
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", profileID=101)`); err != nil {
} else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", columnID=101)`); err != nil {
t.Fatal(err)
}
// Set profile attributes.
if _, err := m.Query("d", "", `SetProfileAttrs(id=100, foo="bar")`); err != nil {
// Set column attributes.
if _, err := m.Query("d", "", `SetColumnAttrs(id=100, foo="bar")`); err != nil {
t.Fatal(err)
}
// Query bitmap.
if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
// Query row.
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -215,16 +215,16 @@ func TestMain_SetProfileAttrs(t *testing.T) {
t.Fatal(err)
}
// Query bitmap after reopening.
if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
// Query row after reopening.
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}
// Ensure program can set profile attributes with columnLabel option.
func TestMain_SetProfileAttrsWithColumnOption(t *testing.T) {
// Ensure program can set column attributes with columnLabel option.
func TestMain_SetColumnAttrsWithColumnOption(t *testing.T) {
m := MustRunMain()
defer m.Close()
@ -236,22 +236,22 @@ func TestMain_SetProfileAttrsWithColumnOption(t *testing.T) {
t.Fatal(err)
}
// Set bits on bitmap.
// Set bits on row.
if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("d", "", `SetBit(id=1, frame="x.n", col=101)`); err != nil {
t.Fatal(err)
}
// Set profile attributes.
if _, err := m.Query("d", "", `SetProfileAttrs(col=100, foo="bar")`); err != nil {
// Set column attributes.
if _, err := m.Query("d", "", `SetColumnAttrs(col=100, foo="bar")`); err != nil {
t.Fatal(err)
}
// Query bitmap.
if res, err := m.Query("d", "profiles=true", `Bitmap(id=1, frame="x.n")`); err != nil {
// Query row.
if res, err := m.Query("d", "columnAttrs=true", `Bitmap(id=1, frame="x.n")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"profiles":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -282,18 +282,18 @@ func TestMain_FrameRestore(t *testing.T) {
// Write data on first cluster.
if _, err := m0.Query("d", "", `
SetBit(id=1, frame="f", profileID=100)
SetBit(id=1, frame="f", profileID=1000)
SetBit(id=1, frame="f", profileID=100000)
SetBit(id=1, frame="f", profileID=200000)
SetBit(id=1, frame="f", profileID=400000)
SetBit(id=1, frame="f", profileID=600000)
SetBit(id=1, frame="f", profileID=800000)
SetBit(id=1, frame="f", columnID=100)
SetBit(id=1, frame="f", columnID=1000)
SetBit(id=1, frame="f", columnID=100000)
SetBit(id=1, frame="f", columnID=200000)
SetBit(id=1, frame="f", columnID=400000)
SetBit(id=1, frame="f", columnID=600000)
SetBit(id=1, frame="f", columnID=800000)
`); err != nil {
t.Fatal(err)
}
// Query bitmap on first cluster.
// Query row on first cluster.
if res, err := m0.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
@ -316,7 +316,7 @@ func TestMain_FrameRestore(t *testing.T) {
t.Fatal(err)
}
// Query bitmap on second cluster.
// Query row on second cluster.
if res, err := m2.Query("d", "", `Bitmap(id=1, frame="f")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
@ -441,14 +441,14 @@ func (m *Main) Query(db, rawQuery, query string) (string, error) {
// SetCommand represents a command to set a bit.
type SetCommand struct {
ID uint64
Frame string
ProfileID uint64
ID uint64
Frame string
ColumnID uint64
}
type SetCommands []SetCommand
// Frames returns the set of profile ids for each frame/bitmap.
// Frames returns the set of column ids for each frame/row.
func (a SetCommands) Frames() map[string]map[uint64][]uint64 {
// Create a set of unique commands.
m := make(map[SetCommand]struct{})
@ -456,16 +456,16 @@ func (a SetCommands) Frames() map[string]map[uint64][]uint64 {
m[cmd] = struct{}{}
}
// Build unique ids for each frame & bitmap.
// Build unique ids for each frame & row.
frames := make(map[string]map[uint64][]uint64)
for cmd := range m {
if frames[cmd.Frame] == nil {
frames[cmd.Frame] = make(map[uint64][]uint64)
}
frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ProfileID)
frames[cmd.Frame][cmd.ID] = append(frames[cmd.Frame][cmd.ID], cmd.ColumnID)
}
// Sort each set of profile ids.
// Sort each set of column ids.
for _, frame := range frames {
for id := range frame {
sort.Sort(uint64Slice(frame[id]))
@ -480,9 +480,9 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand {
cmds := make([]SetCommand, rand.Intn(n))
for i := range cmds {
cmds[i] = SetCommand{
ID: uint64(rand.Intn(1000)),
Frame: "x.n",
ProfileID: uint64(rand.Intn(10)),
ID: uint64(rand.Intn(1000)),
Frame: "x.n",
ColumnID: uint64(rand.Intn(10)),
}
}
return cmds

20
view.go
View file

@ -38,8 +38,8 @@ type View struct {
stats StatsClient
BitmapAttrStore *AttrStore
LogOutput io.Writer
RowAttrStore *AttrStore
LogOutput io.Writer
}
// NewView returns a new instance of View.
@ -124,7 +124,7 @@ func (v *View) openFragments() error {
if err := frag.Open(); err != nil {
return fmt.Errorf("open fragment: slice=%s, err=%s", frag.Slice(), err)
}
frag.BitmapAttrStore = v.BitmapAttrStore
frag.RowAttrStore = v.RowAttrStore
v.fragments[frag.Slice()] = frag
v.stats.Count("maxSlice", 1)
@ -205,7 +205,7 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
if err := frag.Open(); err != nil {
return nil, err
}
frag.BitmapAttrStore = v.BitmapAttrStore
frag.RowAttrStore = v.RowAttrStore
// Save to lookup.
v.fragments[slice] = frag
@ -225,23 +225,23 @@ func (v *View) newFragment(path string, slice uint64) *Fragment {
}
// SetBit sets a bit within the view.
func (v *View) SetBit(bitmapID, profileID uint64) (changed bool, err error) {
slice := profileID / SliceWidth
func (v *View) SetBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
}
return frag.SetBit(bitmapID, profileID)
return frag.SetBit(rowID, columnID)
}
// ClearBit clears a bit within the view.
func (v *View) ClearBit(bitmapID, profileID uint64) (changed bool, err error) {
slice := profileID / SliceWidth
func (v *View) ClearBit(rowID, columnID uint64) (changed bool, err error) {
slice := columnID / SliceWidth
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return changed, err
}
return frag.ClearBit(bitmapID, profileID)
return frag.ClearBit(rowID, columnID)
}
// IsInverseView returns true if the view is used for storing an inverted representation.

28
view_test.go
View file

@ -10,7 +10,7 @@ import (
// View is a test wrapper for pilosa.View.
type View struct {
*pilosa.View
BitmapAttrStore *AttrStore
RowAttrStore *AttrStore
}
// NewView returns a new instance of View with a temporary path.
@ -22,10 +22,10 @@ func NewView(db, frame, name string) *View {
file.Close()
v := &View{
View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize),
BitmapAttrStore: MustOpenAttrStore(),
View: pilosa.NewView(file.Name(), db, frame, name, pilosa.DefaultCacheSize),
RowAttrStore: MustOpenAttrStore(),
}
v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore
v.View.RowAttrStore = v.RowAttrStore.AttrStore
return v
}
@ -41,7 +41,7 @@ func MustOpenView(db, frame, name string) *View {
// Close closes the view and removes all underlying data.
func (v *View) Close() error {
defer os.Remove(v.Path())
defer v.BitmapAttrStore.Close()
defer v.RowAttrStore.Close()
return v.View.Close()
}
@ -53,27 +53,27 @@ func (v *View) Reopen() error {
}
v.View = pilosa.NewView(path, v.DB(), v.Frame(), v.Name(), pilosa.DefaultCacheSize)
v.View.BitmapAttrStore = v.BitmapAttrStore.AttrStore
v.View.RowAttrStore = v.RowAttrStore.AttrStore
if err := v.Open(); err != nil {
return err
}
return nil
}
// MustSetBits sets bits on a bitmap. Panic on error.
// MustSetBits sets bits on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (v *View) MustSetBits(bitmapID uint64, profileIDs ...uint64) {
for _, profileID := range profileIDs {
if _, err := v.SetBit(bitmapID, profileID); err != nil {
func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := v.SetBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// MustClearBits clears bits on a bitmap. Panic on error.
func (v *View) MustClearBits(bitmapID uint64, profileIDs ...uint64) {
for _, profileID := range profileIDs {
if _, err := v.ClearBit(bitmapID, profileID); err != nil {
// MustClearBits clears bits on a row. Panic on error.
func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := v.ClearBit(rowID, columnID); err != nil {
panic(err)
}
}