scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 02:44:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into cors-support

Browse source
This commit is contained in:
Cody Soyland 2018-05-29 08:18:26 -05:00
commit 77a5c6f811
44 changed files with 515 additions and 1456 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

23
api.go
View file

@ -99,9 +99,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
return resp, errors.Wrap(err, "parsing")
}
execOpts := &ExecOptions{
Remote: req.Remote,
ExcludeAttrs: req.ExcludeAttrs,
ExcludeBits: req.ExcludeBits,
Remote: req.Remote,
ExcludeRowAttrs: req.ExcludeRowAttrs,
ExcludeColumns: req.ExcludeColumns,
}
results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts)
if err != nil {
@ -110,7 +110,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
resp.Results = results
// Fill column attributes if requested.
if req.ColumnAttrs && !req.ExcludeBits {
if req.ColumnAttrs && !req.ExcludeColumns {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
@ -118,7 +118,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
if !ok {
continue
}
columnIDs = uint64Slice(columnIDs).merge(bm.Bits())
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
}
// Retrieve column attributes across all calls.
@ -305,7 +305,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin
// Wrap writer with a CSV writer.
cw := csv.NewWriter(w)
// Iterate over each bit.
// Iterate over each column.
if err := f.ForEachBit(func(rowID, columnID uint64) error {
return cw.Write([]string{
strconv.FormatUint(rowID, 10),
@ -784,7 +784,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
// Import into fragment.
err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -803,7 +803,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
// Import into fragment.
err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -813,12 +813,6 @@ func (api *API) MaxSlices(ctx context.Context) map[string]uint64 {
return api.Holder.MaxSlices()
}
// MaxInverseSlices returns the maximum inverse slice number for each index in a
// map.
func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 {
return api.Holder.MaxInverseSlices()
}
// StatsWithTags returns an instance of whatever implementation of StatsClient
// pilosa is using with the given tags.
func (api *API) StatsWithTags(tags []string) StatsClient {
@ -968,7 +962,6 @@ const (
//apiLocalID // not implemented
//apiLongQueryTime // not implemented
apiMarshalFragment
//apiMaxInverseSlices // not implemented
//apiMaxSlices // not implemented
apiQuery
apiRecalculateCaches

4
cache.go
View file

@ -168,7 +168,7 @@ func NewRankCache(maxEntries uint32) *RankCache {
func (c *RankCache) Add(id uint64, n uint64) {
c.mu.Lock()
defer c.mu.Unlock()
// Ignore if the bit count is below the threshold.
// Ignore if the column count is below the threshold.
if n < c.thresholdValue {
return
}
@ -469,7 +469,7 @@ type BitmapCache interface {
// SimpleCache implements BitmapCache
// it is meant to be a short-lived cache for cases where writes are continuing to access
// the same bit within a short time frame (i.e. good for write-heavy loads)
// the same row within a short time frame (i.e. good for write-heavy loads)
// A read-heavy use case would cause the cache to get bigger, potentially causing the
// node to run out of memory.
type SimpleCache struct {

36
client.go
View file

@ -77,16 +77,11 @@ func (c *InternalHTTPClient) Host() *URI { return c.defaultURI }
// MaxSliceByIndex returns the number of slices on a server by index.
func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByIndex(ctx, false)
}
// MaxInverseSliceByIndex returns the number of inverse slices on a server by index.
func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByIndex(ctx, true)
return c.maxSliceByIndex(ctx)
}
// maxSliceByIndex returns the number of slices on a server by index.
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
// Execute request against the host.
u := uriPathToURL(c.defaultURI, "/slices/max")
@ -112,9 +107,6 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool)
return nil, fmt.Errorf("json decode: %s", err)
}
if inverse {
return rsp.Inverse, nil
}
return rsp.Standard, nil
}
@ -308,15 +300,15 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl
return nil
}
// ImportK bulk imports bits to a host.
func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, bits []Bit) error {
// ImportK bulk imports bits specified by string keys to a host.
func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
buf, err := marshalImportPayloadK(index, frame, bits)
buf, err := marshalImportPayloadK(index, frame, columns)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -356,7 +348,7 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte
columnIDs := Bits(bits).ColumnIDs()
timestamps := Bits(bits).Timestamps()
// Marshal bits to protobufs.
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
Index: index,
Frame: frame,
@ -378,7 +370,7 @@ func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) {
columnKeys := Bits(bits).ColumnKeys()
timestamps := Bits(bits).Timestamps()
// Marshal bits to protobufs.
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
Index: index,
Frame: frame,
@ -465,7 +457,7 @@ func marshalImportValuePayload(index, frame, field string, slice uint64, vals []
columnIDs := FieldValues(vals).ColumnIDs()
values := FieldValues(vals).Values()
// Marshal bits to protobufs.
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportValueRequest{
Index: index,
Frame: frame,
@ -524,7 +516,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
} else if !(view == ViewStandard || view == ViewInverse) {
} else if view != ViewStandard {
return ErrInvalidView
}
@ -605,8 +597,6 @@ func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, f
var err error
if view == ViewStandard {
maxSlices, err = c.MaxSliceByIndex(ctx)
} else if view == ViewInverse {
maxSlices, err = c.MaxInverseSliceByIndex(ctx)
} else {
return ErrInvalidView
}
@ -1140,7 +1130,8 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto
return nil
}
// Bit represents the location of a single bit.
// Bit represents the intersection of a row and a column. It can be specifed by
// integer ids or string keys.
type Bit struct {
RowID uint64
ColumnID uint64
@ -1149,7 +1140,7 @@ type Bit struct {
Timestamp int64
}
// Bits represents a slice of bits.
// Bits is a slice of Bit.
type Bits []Bit
func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
@ -1277,7 +1268,7 @@ func (p FieldValues) GroupBySlice() map[uint64][]FieldValue {
return m
}
// BitsByPos represents a slice of bits sorted by internal position.
// BitsByPos is a slice of bits sorted row then column.
type BitsByPos []Bit
func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
@ -1314,7 +1305,6 @@ func nodePathToURL(node *Node, path string) url.URL {
// I don't want to let it go unquestioned.
type InternalClient interface {
MaxSliceByIndex(ctx context.Context) (map[string]uint64, error)
MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error)
Schema(ctx context.Context) ([]*IndexInfo, error)
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error)

135
client_test.go
View file

@ -232,65 +232,11 @@ func TestClient_Import(t *testing.T) {
}
// Verify data.
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) {
t.Fatalf("unexpected bits: %+v", a)
}
}
// Ensure client can bulk import data to an inverse frame.
func TestClient_ImportInverseEnabled(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frameOpts := pilosa.FrameOptions{
InverseEnabled: true,
}
frame, err := idx.CreateFrameIfNotExists("f", frameOpts)
if err != nil {
panic(err)
}
v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse)
if err != nil {
panic(err)
}
f, err := v.CreateFragmentIfNotExists(0)
if err != nil {
panic(err)
}
// Load bitmap into cache to ensure cache gets updated.
f.Row(0)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
{RowID: 200, ColumnID: 5},
{RowID: 200, ColumnID: 6},
}); err != nil {
t.Fatal(err)
}
// Verify data.
if a := f.Row(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected bits: %+v", a)
}
if a := f.Row(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) {
t.Fatalf("unexpected bits: %+v", a)
}
if a := f.Row(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
@ -403,77 +349,20 @@ func TestClient_BackupRestore(t *testing.T) {
}
// Verify data.
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
t.Fatalf("unexpected bits(0): %+v", a)
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) {
t.Fatalf("unexpected bits: %+v", a)
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Columns(); !reflect.DeepEqual(a, []uint64{20000}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
// Ensure client backup and restore a frame with inverse view.
func TestClient_BackupInverseView(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frameOpts := pilosa.FrameOptions{
InverseEnabled: true,
}
frame, err := idx.CreateFrameIfNotExists("f", frameOpts)
if err != nil {
panic(err)
}
v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse)
if err != nil {
panic(err)
}
f, err := v.CreateFragmentIfNotExists(0)
if err != nil {
panic(err)
}
f.SetBit(100, 1)
f.SetBit(100, 2)
f.SetBit(100, 3)
f.SetBit(100, SliceWidth-1)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
// Backup from frame.
var buf bytes.Buffer
if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewInverse); err != nil {
t.Fatal(err)
}
// Restore to a different frame.
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
}
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewInverse); err != nil {
t.Fatal(err)
}
// Verify data.
if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
}
// backup returns error with invalid view
func TestClient_BackupInvalidView(t *testing.T) {
hldr := test.MustOpenHolder()

13
cluster.go
View file

@ -626,21 +626,14 @@ func (a viewsByFrame) addView(frame, view string) {
func (c *Cluster) fragsByHost(idx *Index) fragsByHost {
// frameViews is a map of frame to slice of views.
frameViews := make(viewsByFrame)
inverseFrameViews := make(viewsByFrame)
for _, frame := range idx.Frames() {
for _, view := range frame.Views() {
if IsInverseView(view.Name()) {
inverseFrameViews.addView(frame.Name(), view.Name())
} else {
frameViews.addView(frame.Name(), view.Name())
}
frameViews.addView(frame.Name(), view.Name())
}
}
std := c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews)
inv := c.fragCombos(idx.Name(), idx.MaxInverseSlice(), inverseFrameViews)
return std.add(inv)
return c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews)
}
// fragCombos returns a map (by uri) of lists of fragments for a given index

2
cluster_test.go
View file

@ -424,8 +424,6 @@ func TestCluster_ResizeStates(t *testing.T) {
// Add Field Data to node0.
if err := tc.CreateFrame("i", "fields", FrameOptions{
InverseEnabled: false,
//CacheType: CacheTypeNone,
Fields: []*Field{
{
Name: "fld0",

1
cmd/export.go
View file

@ -53,7 +53,6 @@ The file does not contain any headers.
flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export")
flags.StringVarP(&Exporter.View, "view", "v", "standard", "View to export - default standard")
flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify)

8
cmd/export_test.go
View file

@ -44,7 +44,6 @@ frame = "f1"
v.Check(cmd.Exporter.Host, "localhost:12345")
v.Check(cmd.Exporter.Index, "myindex")
v.Check(cmd.Exporter.Frame, "f1")
v.Check(cmd.Exporter.View, "standard")
v.Check(cmd.Exporter.Path, "/somefile")
return v.Error()
},
@ -52,10 +51,3 @@ frame = "f1"
}
executeDry(t, tests)
}
func TestExportInvalidView(t *testing.T) {
output, err := ExecNewRootCommand(t, "export", "-i", "foo", "-f", "bar", "-v", "test")
if !strings.Contains(err.Error(), "invalid view") {
t.Fatalf("Command 'export' with invalid view should error but: err: '%v', output: '%v'", err, output)
}
}

3
cmd/import.go
View file

@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
importCmd := &cobra.Command{
Use: "import",
Short: "Bulk load data into pilosa.",
Long: `Bulk imports one or more CSV files to a host's index and frame. The bits
Long: `Bulk imports one or more CSV files to a host's index and frame. The data
of the CSV file are grouped by slice for the most efficient import.
The format of the CSV file is:
@ -61,7 +61,6 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")
flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.")
flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked")
flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame")
ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)

14
ctl/export.go
View file

@ -33,7 +33,7 @@ type ExportCommand struct {
// Name of the index & frame to export from.
Index string
Frame string
View string
// Filename to export to.
Path string
@ -59,8 +59,6 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
} else if !(cmd.View == pilosa.ViewStandard || cmd.View == pilosa.ViewInverse) {
return pilosa.ErrInvalidView
}
// Use output file, if specified.
@ -83,13 +81,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
}
// Determine slice count.
var maxSlices map[string]uint64
if cmd.View == pilosa.ViewStandard {
maxSlices, err = client.MaxSliceByIndex(ctx)
} else if cmd.View == pilosa.ViewInverse {
maxSlices, err = client.MaxInverseSliceByIndex(ctx)
}
maxSlices, err := client.MaxSliceByIndex(ctx)
if err != nil {
return errors.Wrap(err, "getting slice count")
}
@ -97,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil {
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, pilosa.ViewStandard, slice, w); err != nil {
return errors.Wrap(err, "exporting")
}
}

8
ctl/export_test.go
View file

@ -41,13 +41,6 @@ func TestExportCommand_Validation(t *testing.T) {
if err != pilosa.ErrFrameRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err)
}
cm.Frame = "f"
cm.View = "test"
err = cm.Run(context.Background())
if err != pilosa.ErrInvalidView {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrInvalidView, err)
}
}
func TestExportCommand_Run(t *testing.T) {
@ -70,7 +63,6 @@ func TestExportCommand_Run(t *testing.T) {
cm.Index = "i"
cm.Frame = "f"
cm.View = pilosa.ViewStandard
if err := cm.Run(context.Background()); err != nil {
t.Fatalf("Export Run doesn't work: %s", err)
}

9
ctl/import.go
View file

@ -107,7 +107,6 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
// Import each path and import by slice.
for _, path := range cmd.Paths {
// Parse path into bits.
logger.Printf("parsing: %s", path)
if err := cmd.importPath(ctx, path); err != nil {
return err
@ -236,13 +235,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
bitsBySlice := pilosa.Bits(bits).GroupBySlice()
// Parse path into bits.
for slice, bits := range bitsBySlice {
for slice, chunk := range bitsBySlice {
if cmd.Sort {
sort.Sort(pilosa.BitsByPos(bits))
sort.Sort(pilosa.BitsByPos(chunk))
}
logger.Printf("importing slice: %d, n=%d", slice, len(bits))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil {
logger.Printf("importing slice: %d, n=%d", slice, len(chunk))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil {
return errors.Wrap(err, "importing")
}
}

2
docs/administration.md
View file

@ -65,7 +65,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count
```
<div class="note">
<p>Note that you must first create a frame with range-encoding enabled and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
<p>Note that you must first create a frame and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
</div>
#### Exporting

1
docs/api-reference.md
View file

@ -105,7 +105,6 @@ The request payload is in JSON, and may contain the `options` field. The `option
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields.
* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
Each individual `field` contains the following:

5
docs/query-language.md
View file

@ -200,14 +200,15 @@ SetColumnAttrs(col=10, url=null)
**Spec:**
```
ClearBit(<frame=STRING>, <row=UINT>, <col=UINT>,
[timestamp=TIMESTAMP])
ClearBit(<frame=STRING>, <row=UINT>, <col=UINT>)
```
**Description:**
`ClearBit` assigns a value of 0 to a bit in the binary matrix, thus disassociating the given row in the given frame from the given column.
Note that clearing bits from time views is not supported.
**Result Type:** boolean
A return value of `true` indicates that the bit was toggled from 1 to 0.

157
executor.go
View file

@ -80,36 +80,21 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// Don't bother calculating slices for query types that don't require it.
needsSlices := needsSlices(q.Calls)
// MaxSlice can differ between inverse and standard views, so we need
// to send queries to different slices based on orientation.
var inverseSlices []uint64
// If slices are specified, then use that value for slices or
// inverseSlices. If slices aren't specified, then include all of them.
if len(slices) > 0 {
// For inverse queries, the values of `slices` provided to the Execute() method
// on the remote node actually represents inverseSlices.
inverseSlices = slices
} else if needsSlices {
// If slices are specified, then use that value for slices. If slices aren't
// specified, then include all of them.
if len(slices) == 0 && needsSlices {
// Round up the number of slices.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
maxSlice := idx.MaxSlice()
maxInverseSlice := idx.MaxInverseSlice()
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
for i := range slices {
slices[i] = uint64(i)
}
// Generate a slices of all inverse slices.
inverseSlices = make([]uint64, maxInverseSlice+1)
for i := range inverseSlices {
inverseSlices[i] = uint64(i)
}
}
// Optimize handling for bulk attribute insertion.
@ -120,23 +105,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
if call.SupportsInverse() && needsSlices {
// Fetch frame & row label based on argument.
frame := call.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
f := e.Holder.Frame(index, frame)
if f == nil {
return nil, ErrFrameNotFound
}
// If this call is to an inverse frame send to a different list of slices.
if call.IsInverse(rowLabel, columnLabel) {
slices = inverseSlices
}
}
v, err := e.executeCall(ctx, index, call, slices, opt)
if err != nil {
return nil, err
@ -336,7 +304,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
// If the row label is used then return bitmap attributes.
row, _ := other.(*Row)
if c.Name == "Bitmap" {
if opt.ExcludeAttrs {
if opt.ExcludeRowAttrs {
row.Attrs = map[string]interface{}{}
} else {
idx := e.Holder.Index(index)
@ -367,7 +335,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
}
}
if opt.ExcludeBits {
if opt.ExcludeColumns {
row.segments = []RowSegment{}
}
@ -580,7 +548,6 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
frame, _ := c.Args["frame"].(string)
inverse, _ := c.Args["inverse"].(bool)
n, _, err := c.UintArg("n")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
@ -619,9 +586,6 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
// Determine view.
view := ViewStandard
if inverse {
view = ViewInverse
}
f := e.Holder.Fragment(index, frame, view, slice)
if f == nil {
@ -685,32 +649,19 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
return nil, ErrFrameNotFound
}
// Return an error if both the row and column label are specified.
rowID, rowOK, rowErr := c.UintArg(rowLabel)
columnID, columnOK, columnErr := c.UintArg(columnLabel)
if rowErr != nil || columnErr != nil {
return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr)
if rowErr != nil {
return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr)
}
if rowOK && columnOK {
return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel)
} else if !rowOK && !columnOK {
return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", rowLabel, columnLabel)
if !rowOK {
return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel)
}
// Determine row or column orientation.
view, id := ViewStandard, rowID
if columnOK {
view, id = ViewInverse, columnID
if !f.InverseEnabled() {
return nil, fmt.Errorf("Bitmap() cannot retrieve columns unless inverse storage enabled")
}
}
frag := e.Holder.Fragment(index, frame, view, slice)
frag := e.Holder.Fragment(index, frame, ViewStandard, slice)
if frag == nil {
return NewRow(), nil
}
return frag.Row(id), nil
return frag.Row(rowID), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
@ -761,26 +712,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Read row & column id.
columnID, columnOK, err := c.UintArg(columnLabel)
if err != nil {
return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err)
}
rowID, rowOK, err := c.UintArg(rowLabel)
if err != nil {
return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err)
}
// Determine view.
var id uint64
var viewName string
if columnOK && rowOK {
return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, rowLabel)
} else if !columnOK && !rowOK {
return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, rowLabel)
} else if columnOK {
viewName, id = ViewInverse, columnID
} else {
viewName, id = ViewStandard, rowID
if !rowOK {
return nil, fmt.Errorf("Range() must specify %q", rowLabel)
}
// Parse start time.
@ -811,12 +748,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Union bitmaps across all time-based subframes.
row := &Row{}
for _, view := range ViewsByTimeRange(viewName, startTime, endTime, q) {
for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(index, frame, view, slice)
if f == nil {
continue
}
row = row.Union(f.Row(id))
row = row.Union(f.Row(rowID))
}
f.Stats.Count("range", 1, 1.0)
return row, nil
@ -1033,7 +970,6 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call,
// executeClearBit executes a ClearBit() call.
func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
view, _ := c.Args["view"].(string)
frame, ok := c.Args["frame"].(string)
if !ok {
return false, errors.New("ClearBit() frame required")
@ -1064,31 +1000,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel)
}
// Clear bits for each view.
switch view {
case ViewStandard:
return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt)
case ViewInverse:
return e.executeClearBitView(ctx, index, c, f, view, rowID, colID, opt)
case "":
var ret bool
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
if f.InverseEnabled() {
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewInverse, rowID, colID, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
}
return ret, nil
default:
return false, fmt.Errorf("invalid view: %s", view)
}
return e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt)
}
// executeClearBitView executes a ClearBit() call for a single view.
@ -1123,7 +1035,6 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql
// executeSetBit executes a SetBit() call.
func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
view, _ := c.Args["view"].(string)
frame, ok := c.Args["frame"].(string)
if !ok {
return false, errors.New("SetBit() field required: frame")
@ -1164,31 +1075,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
timestamp = &t
}
// Set bits for each view.
switch view {
case ViewStandard:
return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt)
case ViewInverse:
return e.executeSetBitView(ctx, index, c, f, view, rowID, colID, timestamp, opt)
case "":
var ret bool
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
if f.InverseEnabled() {
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
}
return ret, nil
default:
return false, fmt.Errorf("invalid view: %s", view)
}
return e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt)
}
// executeSetBitView executes a SetBit() call for a specific view.
@ -1319,7 +1206,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
return err
}
frame.Stats.Count("SetBitmapAttrs", 1, 1.0)
frame.Stats.Count("SetRowAttrs", 1, 1.0)
// Do not forward call if this is already being forwarded.
if opt.Remote {
@ -1404,7 +1291,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil {
return nil, err
}
frame.Stats.Count("SetBitmapAttrs", 1, 1.0)
frame.Stats.Count("SetRowAttrs", 1, 1.0)
}
// Do not forward call if this is already being forwarded.
@ -1700,9 +1587,9 @@ type mapResponse struct {
// ExecOptions represents an execution context for a single Execute() call.
type ExecOptions struct {
Remote bool
ExcludeAttrs bool
ExcludeBits bool
Remote bool
ExcludeRowAttrs bool
ExcludeColumns bool
}
// decodeError returns an error representation of s if s is non-blank.

146
executor_test.go
View file

@ -33,7 +33,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
f, err := index.CreateFrame("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
@ -54,26 +54,26 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit bits.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil {
// Inhibit column attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil {
// Inhibit row attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
@ -83,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if _, err := index.CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -100,14 +100,6 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
})
}
@ -124,8 +116,8 @@ func TestExecutor_Execute_Difference(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -156,8 +148,8 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -186,8 +178,8 @@ func TestExecutor_Execute_Union(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -200,8 +192,8 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -219,8 +211,8 @@ func TestExecutor_Execute_Xor(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -255,7 +247,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatal(err)
} else {
if !res[0].(bool) {
t.Fatalf("expected bit changed")
t.Fatalf("expected column changed")
}
}
@ -266,7 +258,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatal(err)
} else {
if res[0].(bool) {
t.Fatalf("expected bit unchanged")
t.Fatalf("expected column unchanged")
}
}
}
@ -409,12 +401,12 @@ func TestExecutor_Execute_TopN(t *testing.T) {
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set bits for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
} else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
@ -431,7 +423,6 @@ func TestExecutor_Execute_TopN(t *testing.T) {
}
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewInverse, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
@ -445,24 +436,13 @@ func TestExecutor_Execute_TopN(t *testing.T) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("Inverse", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{ID: SliceWidth, Count: 3},
{ID: 0, Count: 2},
}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
}
func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two slices.
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
@ -520,7 +500,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two slices.
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
@ -609,7 +589,6 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100},
},
@ -759,13 +738,12 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
InverseEnabled: true,
TimeQuantum: pilosa.TimeQuantum("YMDH"),
TimeQuantum: pilosa.TimeQuantum("YMDH"),
}); err != nil {
t.Fatal(err)
}
// Set bits.
// Set columns.
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00")
SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00")
@ -784,19 +762,11 @@ func TestExecutor_Execute_Range(t *testing.T) {
t.Run("Standard", func(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("Inverse", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) {
t.Fatalf("unexpected bits: %+v", bits)
}
})
}
// Ensure a Range(field) query can be executed.
@ -854,7 +824,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("EQ", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -863,28 +833,28 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
// NEQ null
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ <int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ -<int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
//t.Fatalf("unexpected result: %s", spew.Sdump(result))
t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Bits())
t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Columns())
}
})
t.Run("LT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -892,7 +862,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("LTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -900,7 +870,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("GT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -908,7 +878,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("GTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -916,7 +886,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("BETWEEN", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -925,7 +895,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("FieldNotNull", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -933,7 +903,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("BelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -941,7 +911,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("AboveMax", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -949,16 +919,16 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("LTAboveMax", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo < 200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits()))
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns()))
}
})
t.Run("GTBelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo > -200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits()))
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns()))
}
})
@ -999,7 +969,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
t.Fatalf("unexpected slices: %+v", slices)
}
// Set bits in slice 0 & 2.
// Set columns in slice 0 & 2.
r := pilosa.NewRow(
(0*SliceWidth)+1,
(0*SliceWidth)+2,
@ -1018,8 +988,8 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -1058,7 +1028,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
}
}
// Ensure a remote query can set bits on multiple nodes.
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
c := test.NewCluster(2)
c.ReplicaN = 2
@ -1101,7 +1071,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
t.Fatal(err)
}
// Verify that one bit is set on both node's holder.
// Verify that one column is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
@ -1110,7 +1080,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
}
// Ensure a remote query can set bits on multiple nodes.
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
c := test.NewCluster(2)
c.ReplicaN = 2
@ -1155,7 +1125,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
t.Fatal(err)
}
// Verify that one bit is set on both node's holder.
// Verify that one column is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
@ -1247,7 +1217,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
index.CreateFrame("f", pilosa.FrameOptions{})
targetAttrs := map[string]interface{}{
"foo": "bar",
}

14
fragment.go
View file

@ -393,7 +393,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
// Determine the position of the bit in the storage.
pos, err := f.pos(rowID, columnID)
if err != nil {
return false, errors.Wrap(err, "getting bit ops")
return false, errors.Wrap(err, "getting bit pos")
}
// Write to storage.
@ -585,7 +585,7 @@ func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uin
// FieldSum returns the sum of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
// Compute count based on the existence bit.
// Compute count based on the existence row.
row := f.Row(uint64(bitDepth))
if filter != nil {
count = row.IntersectionCount(filter)
@ -629,7 +629,7 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
ii := i - 1 // allow for uint range: (bitDepth-1) to 0
row := f.Row(uint64(ii))
x := consider.Difference(row)
@ -662,7 +662,7 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
ii := i - 1 // allow for uint range: (bitDepth-1) to 0
row := f.Row(uint64(ii))
x := row.Intersect(consider)
@ -842,7 +842,7 @@ func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax u
}
// LTE predicateMin
// If bit is zero then remove all set columns not in excluded bitmap.
// If bit is zero then remove all set bits not in excluded bitmap.
if bit2 == 0 {
b = b.Difference(row.Difference(keep2))
} else {
@ -996,13 +996,13 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// If it's too low then don't try finding anymore pairs.
threshold := results.Pairs[0].Count
// If the row doesn't have enough bits set before the intersection
// If the row doesn't have enough columns set before the intersection
// then we can assume that any remaining rows also have a count too low.
if threshold < opt.MinThreshold || cnt < threshold {
break
}
// Calculate the intersecting bit count and skip if it's below our
// Calculate the intersecting column count and skip if it's below our
// last row in our current result set.
count := opt.Src.IntersectionCount(f.Row(rowID))
if count < threshold {

92
fragment_test.go
View file

@ -176,10 +176,10 @@ func TestFragment_SetFieldValue(t *testing.T) {
})
t.Run("QuickCheck", func(t *testing.T) {
if err := quick.Check(func(bitDepth uint, columnN uint64, values []uint64) bool {
if err := quick.Check(func(bitDepth uint, bitN uint64, values []uint64) bool {
// Limit bit depth & maximum values.
bitDepth = (bitDepth % 62) + 1
columnN = (columnN % 99) + 1
bitN = (bitN % 99) + 1
for i := range values {
values[i] = values[i] % (1 << bitDepth)
@ -191,7 +191,7 @@ func TestFragment_SetFieldValue(t *testing.T) {
// Set values.
m := make(map[uint64]int64)
for _, value := range values {
columnID := value % columnN
columnID := value % bitN
m[columnID] = int64(value)
@ -206,9 +206,9 @@ func TestFragment_SetFieldValue(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if value != int64(v) {
t.Fatalf("value mismatch: column=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v)
t.Fatalf("value mismatch: columnID=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v)
} else if !exists {
t.Fatalf("value should exist: column=%d", columnID)
t.Fatalf("value should exist: columnID=%d", columnID)
}
}
@ -353,8 +353,8 @@ func TestFragment_FieldRange(t *testing.T) {
// Query for equality.
if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -376,8 +376,8 @@ func TestFragment_FieldRange(t *testing.T) {
// Query for inequality.
if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -400,32 +400,32 @@ func TestFragment_FieldRange(t *testing.T) {
t.Fatal(err)
}
// Query for fields less than (ending with set bit).
// Query for fields less than (ending with set column).
if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields less than (ending with unset bit).
// Query for fields less than (ending with unset column).
if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields less than or equal to (ending with set bit).
// Query for fields less than or equal to (ending with set column).
if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields less than or equal to (ending with unset bit).
// Query for fields less than or equal to (ending with unset column).
if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -451,29 +451,29 @@ func TestFragment_FieldRange(t *testing.T) {
// Query for fields greater than (ending with unset bit).
if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than (ending with set bit).
if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with unset bit).
if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with set bit).
if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -496,32 +496,32 @@ func TestFragment_FieldRange(t *testing.T) {
t.Fatal(err)
}
// Query for fields greater than (ending with unset bit).
// Query for fields greater than (ending with unset column).
if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than (ending with set bit).
// Query for fields greater than (ending with set column).
if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with unset bit).
// Query for fields greater than or equal to (ending with unset column).
if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with set bit).
// Query for fields greater than or equal to (ending with set column).
if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
}
@ -663,7 +663,7 @@ func TestFragment_TopN_Intersect(t *testing.T) {
}
}
// Ensure a fragment can return top rows that have many bits set.
// Ensure a fragment can return top rows that have many columns set.
func TestFragment_TopN_Intersect_Large(t *testing.T) {
if testing.Short() {
t.Skip("short mode")
@ -1011,8 +1011,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
}
// Verify data in other fragment.
if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected columns: %+v", a)
}
// Close and reopen the fragment & verify the data.
@ -1020,8 +1020,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
t.Fatal(err)
} else if n := f1.Cache().Len(); n != 1 {
t.Fatalf("unexpected cache size (reopen): %d", n)
} else if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected bits (reopen): %+v", a)
} else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected columns (reopen): %+v", a)
}
}

110
frame.go
View file

@ -31,8 +31,7 @@ import (
// Default frame settings.
const (
DefaultCacheType = CacheTypeRanked
DefaultInverseEnabled = false
DefaultCacheType = CacheTypeRanked
// Default ranked frame cache
DefaultCacheSize = 50000
@ -54,11 +53,10 @@ type Frame struct {
Stats StatsClient
// Frame options.
inverseEnabled bool
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
fields []*Field
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
fields []*Field
Logger Logger
}
@ -82,9 +80,8 @@ func NewFrame(path, index, name string) (*Frame, error) {
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
inverseEnabled: DefaultInverseEnabled,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
//timeQuantum
//fields
@ -111,37 +108,18 @@ func (f *Frame) MaxSlice() uint64 {
var max uint64
for _, view := range f.views {
if view.name == ViewInverse {
continue
} else if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max {
if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max {
max = viewMaxSlice
}
}
return max
}
// MaxInverseSlice returns the max inverse slice in the frame.
func (f *Frame) MaxInverseSlice() uint64 {
f.mu.RLock()
defer f.mu.RUnlock()
view := f.views[ViewInverse]
if view == nil {
return 0
}
return view.MaxSlice()
}
// CacheType returns the caching mode for the frame.
func (f *Frame) CacheType() string {
return f.cacheType
}
// InverseEnabled returns true if an inverse view is available.
func (f *Frame) InverseEnabled() bool {
return f.inverseEnabled
}
// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update.
// defaults to DefaultCacheSize 50000
func (f *Frame) SetCacheSize(v uint32) error {
@ -179,11 +157,10 @@ func (f *Frame) Options() FrameOptions {
func (f *Frame) options() FrameOptions {
return FrameOptions{
InverseEnabled: f.inverseEnabled,
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
Fields: f.fields,
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
Fields: f.fields,
}
}
@ -255,7 +232,6 @@ func (f *Frame) loadMeta() error {
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta"))
if os.IsNotExist(err) {
f.inverseEnabled = DefaultInverseEnabled
f.cacheType = DefaultCacheType
f.cacheSize = DefaultCacheSize
f.timeQuantum = ""
@ -270,7 +246,6 @@ func (f *Frame) loadMeta() error {
}
// Copy metadata fields.
f.inverseEnabled = pb.InverseEnabled
f.cacheType = pb.CacheType
if f.cacheType == "" {
f.cacheType = DefaultCacheType
@ -532,11 +507,6 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
// createViewIfNotExistsBase returns the named view, creating it if necessary.
// The returned bool indicates whether the view was created or not.
func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) {
// Don't create inverse views if they are not enabled.
if !f.InverseEnabled() && IsInverseView(name) {
return nil, false, ErrFrameInverseDisabled
}
f.mu.Lock()
defer f.mu.Unlock()
@ -840,16 +810,14 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
timestamp = timestamps[i]
}
var standard, inverse []string
var standard []string
if timestamp == nil {
standard = []string{ViewStandard}
inverse = []string{ViewInverse}
} else {
standard = ViewsByTime(ViewStandard, *timestamp, q)
// In order to match the logic of `SetBit()`, we want bits
// with timestamps to write to both time and standard views.
standard = append(standard, ViewStandard)
inverse = ViewsByTime(ViewInverse, *timestamp, q)
}
// Attach bit to each standard view.
@ -860,34 +828,10 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
data.ColumnIDs = append(data.ColumnIDs, columnID)
dataByFragment[key] = data
}
if f.inverseEnabled {
// Attach reversed bits to each inverse view.
for _, name := range inverse {
key := importKey{View: name, Slice: rowID / SliceWidth}
data := dataByFragment[key]
data.RowIDs = append(data.RowIDs, columnID) // reversed
data.ColumnIDs = append(data.ColumnIDs, rowID) // reversed
dataByFragment[key] = data
}
}
}
// Import into each fragment.
for key, data := range dataByFragment {
// Skip inverse data if inverse is not enabled.
if !f.inverseEnabled && IsInverseView(key.View) {
continue
}
// Re-sort data for inverse views.
if IsInverseView(key.View) {
sort.Sort(importBitSet{
rowIDs: data.RowIDs,
columnIDs: data.ColumnIDs,
})
}
view, err := f.CreateViewIfNotExists(key.View)
if err != nil {
return errors.Wrap(err, "creating view")
@ -1003,12 +947,10 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FrameOptions represents options to set when initializing a frame.
type FrameOptions struct {
InverseEnabled bool `json:"inverseEnabled,omitempty"`
RangeEnabled bool `json:"rangeEnabled,omitempty"` // deprecated, will be removed
CacheType string `json:"cacheType,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Fields []*Field `json:"fields,omitempty"`
CacheType string `json:"cacheType,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Fields []*Field `json:"fields,omitempty"`
}
// Encode converts o into its internal representation.
@ -1021,11 +963,10 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta {
return nil
}
return &internal.FrameMeta{
InverseEnabled: o.InverseEnabled,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
TimeQuantum: string(o.TimeQuantum),
Fields: encodeFields(o.Fields),
CacheType: o.CacheType,
CacheSize: o.CacheSize,
TimeQuantum: string(o.TimeQuantum),
Fields: encodeFields(o.Fields),
}
}
@ -1034,11 +975,10 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions {
return nil
}
return &FrameOptions{
InverseEnabled: options.InverseEnabled,
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Fields: decodeFields(options.Fields),
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Fields: decodeFields(options.Fields),
}
}

34
handler.go
View file

@ -106,8 +106,8 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) {
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")
h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse")
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits")
h.validators["GetSliceMax"] = queryValidationSpecRequired()
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice")
@ -328,7 +328,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{
Standard: h.API.MaxSlices(r.Context()),
Inverse: h.API.MaxInverseSlices(r.Context()),
}); err != nil {
h.Logger.Printf("write slices-max response error: %s", err)
}
@ -336,7 +335,6 @@ func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
type getSlicesMaxResponse struct {
Standard map[string]uint64 `json:"standard"`
Inverse map[string]uint64 `json:"inverse"`
}
// handleGetIndexes handles GET /index request.
@ -846,11 +844,11 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
}
return &QueryRequest{
Query: query,
Slices: slices,
ColumnAttrs: q.Get("columnAttrs") == "true",
ExcludeAttrs: q.Get("excludeAttrs") == "true",
ExcludeBits: q.Get("excludeBits") == "true",
Query: query,
Slices: slices,
ColumnAttrs: q.Get("columnAttrs") == "true",
ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true",
ExcludeColumns: q.Get("excludeColumns") == "true",
}, nil
}
@ -1206,10 +1204,10 @@ type QueryRequest struct {
ColumnAttrs bool
// Do not return row attributes, if true.
ExcludeAttrs bool
ExcludeRowAttrs bool
// Do not return bits, if true.
ExcludeBits bool
// Do not return columns, if true.
ExcludeColumns bool
// If true, indicates that query is part of a larger distributed query.
// If false, this request is on the originating node.
@ -1218,12 +1216,12 @@ type QueryRequest struct {
func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest {
req := &QueryRequest{
Query: pb.Query,
Slices: pb.Slices,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeAttrs: pb.ExcludeAttrs,
ExcludeBits: pb.ExcludeBits,
Query: pb.Query,
Slices: pb.Slices,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeRowAttrs: pb.ExcludeRowAttrs,
ExcludeColumns: pb.ExcludeColumns,
}
return req

4
handler_internal_test.go
View file

@ -66,8 +66,8 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) {
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
{json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}},
{json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}},
{json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"},
{json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}},
{json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
}
for _, test := range tests {

71
handler_test.go
View file

@ -84,12 +84,10 @@ func TestHandler_Schema(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
@ -107,8 +105,8 @@ func TestHandler_Schema(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -123,12 +121,10 @@ func TestHandler_Status(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
@ -209,48 +205,7 @@ func TestHandler_MaxSlices(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can return the maxslice map for the inverse views.
func TestHandler_MaxSlices_Inverse(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true})
if err != nil {
t.Fatal(err)
}
if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+1, nil); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+2, nil); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (3*SliceWidth)+4, nil); err != nil {
t.Fatal(err)
}
f1, err := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true})
if err != nil {
t.Fatal(err)
}
if _, err := f1.SetBit(pilosa.ViewStandard, 40, (0*SliceWidth)+1, nil); err != nil {
t.Fatal(err)
} else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+2, nil); err != nil {
t.Fatal(err)
} else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+4, nil); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -419,7 +374,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -452,7 +407,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -484,8 +439,8 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
@ -541,8 +496,8 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
@ -1145,8 +1100,8 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0)
if f1 == nil {
t.Fatal("fragment x/y/standard/0 not created")
} else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) {
t.Fatalf("unexpected restored bits: %+v", bits)
} else if columns := f1.Row(100).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 3}) {
t.Fatalf("unexpected restored columns: %+v", columns)
}
}

10
holder.go
View file

@ -209,15 +209,6 @@ func (h *Holder) MaxSlices() map[string]uint64 {
return a
}
// MaxInverseSlices returns MaxInverseSlice map for all indexes.
func (h *Holder) MaxInverseSlices() map[string]uint64 {
a := make(map[string]uint64)
for _, index := range h.Indexes() {
a[index.Name()] = index.MaxInverseSlice()
}
return a
}
// Schema returns schema information for all indexes, frames, and views.
func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
@ -270,7 +261,6 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
func (h *Holder) EncodeMaxSlices() *internal.MaxSlices {
return &internal.MaxSlices{
Standard: h.MaxSlices(),
Inverse: h.MaxInverseSlices(),
}
}

88
holder_test.go
View file

@ -456,29 +456,29 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
}
@ -554,29 +554,29 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
@ -597,16 +597,16 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
@ -615,8 +615,8 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
}

37
index.go
View file

@ -38,8 +38,7 @@ type Index struct {
frames map[string]*Frame
// Max Slice on any node in the cluster, according to this node.
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
remoteMaxSlice uint64
NewAttrStore func(string) AttrStore
@ -64,8 +63,7 @@ func NewIndex(path, name string) (*Index, error) {
name: name,
frames: make(map[string]*Frame),
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
remoteMaxSlice: 0,
NewAttrStore: NewNopAttrStore,
columnAttrStore: NopAttrStore,
@ -236,30 +234,6 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) {
i.remoteMaxSlice = newmax
}
// MaxInverseSlice returns the max inverse slice in the index according to this node.
func (i *Index) MaxInverseSlice() uint64 {
if i == nil {
return 0
}
i.mu.RLock()
defer i.mu.RUnlock()
max := i.remoteMaxInverseSlice
for _, f := range i.frames {
if slice := f.MaxInverseSlice(); slice > max {
max = slice
}
}
return max
}
// SetRemoteMaxInverseSlice sets the remote max inverse slice value received from another node.
func (i *Index) SetRemoteMaxInverseSlice(v uint64) {
i.mu.Lock()
defer i.mu.Unlock()
i.remoteMaxInverseSlice = v
}
// FramePath returns the path to a frame in the index.
func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
@ -325,11 +299,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
return nil, ErrInvalidCacheType
}
// Validate mutually exclusive options if ranges are enabled.
if opt.RangeEnabled {
i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame")
}
// Validate fields.
for _, field := range opt.Fields {
if err := ValidateField(field); err != nil {
@ -364,8 +333,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
f.cacheSize = opt.CacheSize
}
f.inverseEnabled = opt.InverseEnabled
// Set fields.
f.fields = opt.Fields

49
index_test.go
View file

@ -67,14 +67,13 @@ func TestIndex_CreateFrame(t *testing.T) {
})
// Ensure frame can include range columns.
t.Run("RangeEnabled", func(t *testing.T) {
t.Run("BSIFields", func(t *testing.T) {
t.Run("OK", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame with schema and verify it exists.
if f, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: false,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
@ -99,49 +98,6 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
t.Run("ErrInverseRangeAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
frame, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
InverseEnabled: true,
Fields: []*pilosa.Field{
&pilosa.Field{
Name: "myfield",
Type: pilosa.FieldTypeInt,
Min: -20,
Max: 100,
},
},
})
if err != nil {
t.Fatal(err)
}
ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetFieldValue(1, "myfield", 87)
if !ch || err != nil {
t.Fatal(ch, err)
}
views := frame.Views()
if len(views) != 3 {
var names string
for _, v := range views {
names = names + v.Name() + " "
}
t.Fatalf("Unexpected views: %s", names)
}
})
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
@ -153,7 +109,7 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
t.Run("RangeEnabledWithCacheTypeNone", func(t *testing.T) {
t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
@ -208,7 +164,6 @@ func TestIndex_CreateFrame(t *testing.T) {
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
},

411
internal/private.pb.go
View file

@ -70,12 +70,10 @@ func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} }
type FrameMeta struct {
InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"`
CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"`
TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
RangeEnabled bool `protobuf:"varint,6,opt,name=RangeEnabled,proto3" json:"RangeEnabled,omitempty"`
Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"`
CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"`
TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"`
}
func (m *FrameMeta) Reset() { *m = FrameMeta{} }
@ -83,13 +81,6 @@ func (m *FrameMeta) String() string { return proto.CompactTextString(
func (*FrameMeta) ProtoMessage() {}
func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} }
func (m *FrameMeta) GetInverseEnabled() bool {
if m != nil {
return m.InverseEnabled
}
return false
}
func (m *FrameMeta) GetCacheType() string {
if m != nil {
return m.CacheType
@ -111,13 +102,6 @@ func (m *FrameMeta) GetTimeQuantum() string {
return ""
}
func (m *FrameMeta) GetRangeEnabled() bool {
if m != nil {
return m.RangeEnabled
}
return false
}
func (m *FrameMeta) GetFields() []*Field {
if m != nil {
return m.Fields
@ -231,7 +215,6 @@ func (m *Cache) GetIDs() []uint64 {
type MaxSlices struct {
Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
Inverse map[string]uint64 `protobuf:"bytes,2,rep,name=Inverse" json:"Inverse,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
}
func (m *MaxSlices) Reset() { *m = MaxSlices{} }
@ -246,17 +229,9 @@ func (m *MaxSlices) GetStandard() map[string]uint64 {
return nil
}
func (m *MaxSlices) GetInverse() map[string]uint64 {
if m != nil {
return m.Inverse
}
return nil
}
type CreateSliceMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
IsInverse bool `protobuf:"varint,3,opt,name=IsInverse,proto3" json:"IsInverse,omitempty"`
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
}
func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} }
@ -278,13 +253,6 @@ func (m *CreateSliceMessage) GetSlice() uint64 {
return 0
}
func (m *CreateSliceMessage) GetIsInverse() bool {
if m != nil {
return m.IsInverse
}
return false
}
type DeleteIndexMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
}
@ -1067,16 +1035,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if m.InverseEnabled {
dAtA[i] = 0x10
i++
if m.InverseEnabled {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if len(m.CacheType) > 0 {
dAtA[i] = 0x1a
i++
@ -1094,16 +1052,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum)))
i += copy(dAtA[i:], m.TimeQuantum)
}
if m.RangeEnabled {
dAtA[i] = 0x30
i++
if m.RangeEnabled {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if len(m.Fields) > 0 {
for _, msg := range m.Fields {
dAtA[i] = 0x3a
@ -1307,22 +1255,6 @@ func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPrivate(dAtA, i, uint64(v))
}
}
if len(m.Inverse) > 0 {
for k, _ := range m.Inverse {
dAtA[i] = 0x12
i++
v := m.Inverse[k]
mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v))
i = encodeVarintPrivate(dAtA, i, uint64(mapSize))
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(k)))
i += copy(dAtA[i:], k)
dAtA[i] = 0x10
i++
i = encodeVarintPrivate(dAtA, i, uint64(v))
}
}
return i, nil
}
@ -1352,16 +1284,6 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Slice))
}
if m.IsInverse {
dAtA[i] = 0x18
i++
if m.IsInverse {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -2324,9 +2246,6 @@ func (m *IndexMeta) Size() (n int) {
func (m *FrameMeta) Size() (n int) {
var l int
_ = l
if m.InverseEnabled {
n += 2
}
l = len(m.CacheType)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
@ -2338,9 +2257,6 @@ func (m *FrameMeta) Size() (n int) {
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.RangeEnabled {
n += 2
}
if len(m.Fields) > 0 {
for _, e := range m.Fields {
l = e.Size()
@ -2428,14 +2344,6 @@ func (m *MaxSlices) Size() (n int) {
n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize))
}
}
if len(m.Inverse) > 0 {
for k, v := range m.Inverse {
_ = k
_ = v
mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v))
n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize))
}
}
return n
}
@ -2449,9 +2357,6 @@ func (m *CreateSliceMessage) Size() (n int) {
if m.Slice != 0 {
n += 1 + sovPrivate(uint64(m.Slice))
}
if m.IsInverse {
n += 2
}
return n
}
@ -2957,26 +2862,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error {
return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.InverseEnabled = bool(v != 0)
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field CacheType", wireType)
@ -3054,26 +2939,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error {
}
m.TimeQuantum = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field RangeEnabled", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.RangeEnabled = bool(v != 0)
case 7:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType)
@ -3802,113 +3667,6 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error {
}
m.Standard[mapkey] = mapvalue
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Inverse", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Inverse == nil {
m.Inverse = make(map[string]uint64)
}
var mapkey string
var mapvalue uint64
for iNdEx < postIndex {
entryPreIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
if fieldNum == 1 {
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthPrivate
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
return io.ErrUnexpectedEOF
}
mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
} else if fieldNum == 2 {
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
mapvalue |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
} else {
iNdEx = entryPreIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > postIndex {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
m.Inverse[mapkey] = mapvalue
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -4007,26 +3765,6 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IsInverse", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.IsInverse = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7259,75 +6997,70 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 1112 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6f, 0x1b, 0x45,
0x14, 0x67, 0xbd, 0x6b, 0x27, 0x7e, 0xae, 0x53, 0x67, 0x5a, 0xca, 0xb6, 0xaa, 0x82, 0x19, 0x15,
0x6a, 0x38, 0x44, 0x25, 0xbd, 0x40, 0xa1, 0x52, 0x95, 0x38, 0x15, 0x8b, 0x48, 0x04, 0xe3, 0xa4,
0x07, 0x24, 0x90, 0x26, 0xf6, 0x28, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0xf9, 0xe8, 0x81, 0x33,
0x17, 0xee, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x01, 0x42, 0xe1, 0x0f, 0x01, 0xbd, 0x37, 0xb3, 0x1f,
0xb1, 0x9d, 0xa6, 0x04, 0x6e, 0xf3, 0x3e, 0xe7, 0xf7, 0x3e, 0x67, 0x17, 0xda, 0x93, 0x34, 0x3c,
0x96, 0x5a, 0xad, 0x4f, 0xd2, 0x44, 0x27, 0x6c, 0x39, 0x8c, 0xb5, 0x4a, 0x63, 0x19, 0xf1, 0x16,
0x34, 0x83, 0x78, 0xa4, 0x4e, 0x77, 0x94, 0x96, 0xfc, 0x0f, 0x07, 0x9a, 0xcf, 0x53, 0x39, 0x56,
0x48, 0xb1, 0x0f, 0x60, 0x25, 0x88, 0x8f, 0x55, 0x9a, 0xa9, 0xed, 0x58, 0x1e, 0x44, 0x6a, 0xe4,
0xd7, 0xba, 0x4e, 0x6f, 0x59, 0xcc, 0x70, 0xd9, 0x7d, 0x68, 0x6e, 0xc9, 0xe1, 0x4b, 0xb5, 0x77,
0x36, 0x51, 0xbe, 0xdb, 0x75, 0x7a, 0x4d, 0x51, 0x32, 0x0a, 0xe9, 0x20, 0x7c, 0xa5, 0x7c, 0xaf,
0xeb, 0xf4, 0xda, 0xa2, 0x64, 0xb0, 0x2e, 0xb4, 0xf6, 0xc2, 0xb1, 0xfa, 0x66, 0x2a, 0x63, 0x3d,
0x1d, 0xfb, 0x75, 0xb2, 0xae, 0xb2, 0x18, 0x87, 0x1b, 0x42, 0xc6, 0x87, 0x05, 0x86, 0x06, 0x61,
0xb8, 0xc0, 0x63, 0x0f, 0xa1, 0xf1, 0x3c, 0x54, 0xd1, 0x28, 0xf3, 0x97, 0xba, 0x6e, 0xaf, 0xb5,
0x71, 0x73, 0x3d, 0x8f, 0x6f, 0x9d, 0xf8, 0xc2, 0x8a, 0x39, 0x87, 0x95, 0x60, 0x3c, 0x49, 0x52,
0x2d, 0x54, 0x36, 0x49, 0xe2, 0x4c, 0xb1, 0x0e, 0xb8, 0xdb, 0x69, 0xea, 0x3b, 0x74, 0x31, 0x1e,
0xf9, 0x8f, 0xd0, 0xd9, 0x8c, 0x92, 0xe1, 0x51, 0x5f, 0x6a, 0x29, 0xd4, 0x0f, 0x53, 0x95, 0x69,
0x76, 0x1b, 0xea, 0x94, 0x25, 0xab, 0x67, 0x08, 0xe4, 0x52, 0xb6, 0x28, 0x2f, 0x4d, 0x61, 0x08,
0xe4, 0x92, 0x3d, 0xa5, 0xc2, 0x13, 0x86, 0x40, 0xee, 0x20, 0x0a, 0x87, 0x26, 0x05, 0x9e, 0x30,
0x04, 0x63, 0xe0, 0xbd, 0x08, 0xd5, 0x89, 0x8d, 0x9b, 0xce, 0x3c, 0x80, 0xd5, 0xca, 0xfd, 0x16,
0xe6, 0x1d, 0x68, 0x88, 0xe4, 0x24, 0xe8, 0x67, 0xbe, 0xd3, 0x75, 0x7b, 0x9e, 0xb0, 0x14, 0x65,
0x37, 0x89, 0xa6, 0xe3, 0x18, 0x45, 0x35, 0x12, 0x95, 0x0c, 0x7e, 0x17, 0xea, 0x94, 0x6a, 0x8c,
0xb2, 0xb4, 0xc5, 0x23, 0xff, 0xdb, 0x81, 0xe6, 0x8e, 0x3c, 0x25, 0x18, 0x19, 0x7b, 0x0a, 0xcb,
0x03, 0x2d, 0xe3, 0x91, 0x4c, 0x47, 0xa4, 0xd4, 0xda, 0x78, 0xaf, 0x4c, 0x61, 0xa1, 0xb6, 0x9e,
0xeb, 0x6c, 0xc7, 0x3a, 0x3d, 0x13, 0x85, 0x09, 0x7b, 0x02, 0x4b, 0xb6, 0x27, 0x08, 0x43, 0x6b,
0xa3, 0xbb, 0xc8, 0xba, 0x68, 0x1b, 0x34, 0xce, 0x0d, 0xee, 0x7d, 0x06, 0xed, 0x0b, 0x6e, 0x11,
0xeb, 0x91, 0x3a, 0xcb, 0x2b, 0x72, 0xa4, 0xce, 0x30, 0x77, 0xc7, 0x32, 0x9a, 0x9a, 0x3c, 0x7b,
0xc2, 0x10, 0x4f, 0x6a, 0x9f, 0x38, 0xf7, 0x9e, 0xc0, 0x8d, 0xaa, 0xd7, 0x7f, 0x63, 0xcb, 0xbf,
0x07, 0xb6, 0x95, 0x2a, 0xa9, 0x15, 0xc1, 0xdb, 0x51, 0x59, 0x26, 0x0f, 0xd5, 0xe5, 0x95, 0x36,
0xd5, 0xab, 0x55, 0xab, 0x77, 0x1f, 0x9a, 0x41, 0x96, 0x07, 0xee, 0x52, 0x5f, 0x96, 0x0c, 0xfe,
0x11, 0xb0, 0xbe, 0x8a, 0x94, 0x56, 0x76, 0xbe, 0x5e, 0xe3, 0x9f, 0x0f, 0x72, 0x2c, 0x57, 0xeb,
0xb2, 0x87, 0xe0, 0xe1, 0x78, 0x12, 0x94, 0xd6, 0xc6, 0xad, 0x32, 0xd3, 0xc5, 0x1c, 0x0b, 0x52,
0xe0, 0x61, 0xee, 0xd4, 0x8e, 0xf4, 0x15, 0x01, 0x2e, 0x68, 0xe5, 0xfc, 0x2a, 0x77, 0xf6, 0xaa,
0x62, 0x49, 0xd8, 0xab, 0x9e, 0xe5, 0xb1, 0x5e, 0xf7, 0x2a, 0x7e, 0x58, 0x80, 0xc5, 0x49, 0xbd,
0x0e, 0xd8, 0xf7, 0xa1, 0x4e, 0xb6, 0x16, 0xed, 0xdc, 0x0e, 0x30, 0x52, 0xfe, 0xa2, 0x80, 0x7a,
0xdd, 0x8b, 0x6e, 0x57, 0x2f, 0x6a, 0xe6, 0x7e, 0xbf, 0xb5, 0xba, 0x38, 0xd3, 0xbb, 0x68, 0x63,
0x3c, 0xd1, 0xf9, 0xf2, 0x9a, 0xcd, 0x24, 0x12, 0x7d, 0xe3, 0x12, 0xc8, 0x7c, 0xb7, 0xeb, 0xa2,
0x6f, 0x22, 0xf8, 0x63, 0x68, 0x0c, 0x86, 0x2f, 0xd5, 0x58, 0xb2, 0x0f, 0x71, 0xd2, 0x46, 0xea,
0x54, 0x65, 0x76, 0x4e, 0x6f, 0xce, 0xd4, 0x5f, 0xe4, 0x72, 0xde, 0xb7, 0x21, 0x5d, 0x02, 0xa8,
0x41, 0x57, 0x67, 0xbe, 0x37, 0xb7, 0x31, 0x91, 0x2f, 0xac, 0x98, 0x6f, 0x83, 0xbb, 0x2f, 0x02,
0xdc, 0x3f, 0x84, 0x20, 0xf7, 0x62, 0x29, 0xf4, 0xfd, 0x45, 0x92, 0x69, 0x9b, 0x20, 0x3a, 0x23,
0xef, 0xeb, 0x24, 0xd5, 0x94, 0x9e, 0xb6, 0xa0, 0x33, 0xff, 0x0e, 0xbc, 0xdd, 0x64, 0xa4, 0xd8,
0x0a, 0xd4, 0x82, 0xbe, 0xf5, 0x51, 0x0b, 0xfa, 0xec, 0x5d, 0x72, 0x6f, 0xf3, 0xd2, 0x2e, 0x41,
0xec, 0x8b, 0x40, 0xd0, 0xc5, 0x0f, 0xa0, 0x1d, 0x64, 0x5b, 0x49, 0x92, 0x8e, 0xc2, 0x58, 0xea,
0x24, 0xb5, 0x73, 0x76, 0x91, 0xc9, 0x9f, 0x41, 0x07, 0xdd, 0x0f, 0xb4, 0xd4, 0x45, 0xf7, 0xdd,
0x81, 0x06, 0xf2, 0x8a, 0xeb, 0x2c, 0x45, 0xb3, 0x8c, 0x7a, 0x79, 0x51, 0x89, 0xe0, 0x5f, 0x19,
0x0f, 0xdb, 0xc7, 0x2a, 0xd6, 0x95, 0xa6, 0x20, 0x9a, 0x1c, 0xb4, 0x85, 0x21, 0x18, 0x37, 0xa1,
0x58, 0xcc, 0x2b, 0x25, 0x66, 0xe4, 0x0a, 0x92, 0xf1, 0x9f, 0x1d, 0x80, 0x1c, 0xd0, 0x34, 0x2b,
0x4c, 0x9c, 0xcb, 0x4d, 0xd8, 0xc7, 0x95, 0x7d, 0x3c, 0xdf, 0x27, 0x85, 0x48, 0x54, 0xb6, 0x76,
0x2f, 0x6f, 0x0b, 0xdb, 0xf2, 0x9d, 0x52, 0xdf, 0xf0, 0x6d, 0x99, 0x70, 0x15, 0xb4, 0xb7, 0xa2,
0x69, 0xa6, 0x55, 0x6a, 0x11, 0xe1, 0xbb, 0x61, 0x18, 0x45, 0x7e, 0x4a, 0xc6, 0xe2, 0x14, 0xb1,
0x07, 0x50, 0x47, 0xa4, 0xa6, 0x37, 0xe7, 0xc3, 0x30, 0x42, 0x3e, 0xb0, 0xd3, 0xb1, 0xb0, 0xed,
0x18, 0x78, 0xf4, 0x95, 0x60, 0xdb, 0x85, 0x3e, 0x10, 0x3a, 0xe0, 0xee, 0x84, 0x31, 0x85, 0xe0,
0x0a, 0x3c, 0x12, 0x47, 0x9e, 0xd2, 0x4b, 0x89, 0x1c, 0x89, 0xfb, 0x71, 0xd5, 0x6c, 0x07, 0x9c,
0x87, 0xeb, 0xcc, 0x6c, 0xfe, 0xd0, 0xba, 0x95, 0x87, 0x76, 0x00, 0xab, 0x66, 0x13, 0xfc, 0x9f,
0x4e, 0x7f, 0xad, 0xc1, 0xaa, 0x50, 0x59, 0xf8, 0x4a, 0x05, 0x71, 0xa6, 0xd3, 0xe9, 0x50, 0x87,
0x49, 0x8c, 0xf6, 0x5f, 0x26, 0x07, 0x36, 0xd5, 0xae, 0x30, 0xc4, 0x9b, 0x74, 0x12, 0x7b, 0x04,
0xad, 0xd9, 0xee, 0x9f, 0x57, 0xad, 0xaa, 0xb0, 0x47, 0xb0, 0x34, 0x48, 0xa6, 0xe9, 0xb0, 0x98,
0xed, 0x3b, 0xa5, 0xb6, 0x41, 0x66, 0xc4, 0x22, 0x57, 0xab, 0xf4, 0x51, 0xfd, 0xf5, 0x7d, 0xc4,
0x9e, 0xce, 0xf4, 0x11, 0x7d, 0x8d, 0xb5, 0x36, 0xde, 0x29, 0x0d, 0x2e, 0x88, 0xc5, 0x45, 0x6d,
0xfe, 0x93, 0x03, 0x37, 0xaa, 0x10, 0xde, 0x68, 0x30, 0x8a, 0x8a, 0xd4, 0x16, 0x56, 0xc4, 0x5d,
0x54, 0x11, 0xaf, 0xac, 0x48, 0xf9, 0x76, 0xd7, 0x2b, 0x6f, 0x37, 0x3f, 0x82, 0xbb, 0x73, 0x65,
0xda, 0x4a, 0xc6, 0x13, 0xec, 0x87, 0xff, 0x50, 0x2e, 0x5c, 0x19, 0x69, 0x6a, 0x0b, 0xd5, 0x14,
0x86, 0xe0, 0x9f, 0xc2, 0xdb, 0x03, 0xa5, 0x2b, 0x45, 0xca, 0xbb, 0xad, 0x0b, 0xee, 0xae, 0x3a,
0xb9, 0x24, 0x7c, 0x14, 0xf1, 0xcf, 0xc1, 0xdf, 0x9f, 0x8c, 0xa4, 0x56, 0xd7, 0xb2, 0xde, 0x84,
0xe5, 0xbd, 0x64, 0x92, 0x44, 0xc9, 0xe1, 0xd9, 0x15, 0x23, 0xef, 0xc3, 0x92, 0xd9, 0x8f, 0xe6,
0x33, 0xb2, 0x29, 0x72, 0x92, 0xdf, 0xc2, 0x86, 0x1e, 0xca, 0x68, 0x38, 0x8d, 0x10, 0x06, 0x7e,
0x4f, 0x66, 0x9b, 0x9d, 0xdf, 0xce, 0xd7, 0x9c, 0xdf, 0xcf, 0xd7, 0x9c, 0x3f, 0xcf, 0xd7, 0x9c,
0x5f, 0xfe, 0x5a, 0x7b, 0xeb, 0xa0, 0x41, 0x7f, 0x16, 0x8f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff,
0xfa, 0xf5, 0x5b, 0x36, 0x6a, 0x0c, 0x00, 0x00,
// 1035 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1b, 0x45,
0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x8c, 0x53, 0x67, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45,
0x0d, 0x1c, 0xa2, 0x92, 0x5e, 0x78, 0x55, 0x8a, 0x12, 0xa7, 0x62, 0x11, 0x89, 0x60, 0x36, 0xe9,
0x01, 0x89, 0xc3, 0xd4, 0x1e, 0xa5, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x1e, 0x3d, 0x70, 0x85,
0x0b, 0x17, 0x4e, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f,
0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xfb, 0xbd, 0x5f, 0xbf, 0xef, 0x9b, 0x85, 0xee, 0x24, 0x8d, 0xce,
0xb9, 0x12, 0xdb, 0x93, 0x54, 0x2a, 0x49, 0x56, 0xa3, 0x44, 0x89, 0x34, 0xe1, 0x31, 0xed, 0x40,
0x3b, 0x48, 0x46, 0xe2, 0xf2, 0x50, 0x28, 0x4e, 0x7f, 0x77, 0xa0, 0xfd, 0x34, 0xe5, 0x63, 0xa1,
0x29, 0xf2, 0x3e, 0xb4, 0xf7, 0xf9, 0xf0, 0x85, 0x38, 0xbe, 0x9a, 0x08, 0xdf, 0xed, 0x3b, 0x5b,
0x6d, 0x56, 0x31, 0x4a, 0x69, 0x18, 0xbd, 0x14, 0xbe, 0xd7, 0x77, 0xb6, 0xba, 0xac, 0x62, 0x90,
0x3e, 0x74, 0x8e, 0xa3, 0xb1, 0xf8, 0x3e, 0xe7, 0x89, 0xca, 0xc7, 0x7e, 0x13, 0xad, 0xeb, 0x2c,
0xf2, 0x10, 0x5a, 0x4f, 0x23, 0x11, 0x8f, 0x32, 0x7f, 0xa5, 0xef, 0x6e, 0x75, 0x76, 0xee, 0x6c,
0x17, 0x39, 0x6d, 0x23, 0x9f, 0x59, 0x31, 0xa5, 0xb0, 0x16, 0x8c, 0x27, 0x32, 0x55, 0x4c, 0x64,
0x13, 0x99, 0x64, 0x82, 0xf4, 0xc0, 0x3d, 0x48, 0x53, 0xdf, 0x41, 0xa7, 0xfa, 0x93, 0xfe, 0x0c,
0xbd, 0xbd, 0x58, 0x0e, 0xcf, 0x06, 0x5c, 0x71, 0x26, 0x7e, 0xca, 0x45, 0xa6, 0xc8, 0x3d, 0x68,
0x62, 0x65, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x15, 0xfa, 0x0d, 0xc3, 0x45, 0x42, 0x73, 0xd1, 0x1e,
0xcb, 0xf4, 0x98, 0x21, 0x34, 0x37, 0x8c, 0xa3, 0xa1, 0x29, 0xcf, 0x63, 0x86, 0x20, 0x04, 0xbc,
0x67, 0x91, 0xb8, 0xb0, 0x35, 0xe1, 0x37, 0x0d, 0x60, 0xbd, 0x16, 0xdf, 0xa6, 0xb9, 0x01, 0x2d,
0x26, 0x2f, 0x82, 0x41, 0xe6, 0x3b, 0x7d, 0x77, 0xcb, 0x63, 0x96, 0xc2, 0xce, 0xc9, 0x38, 0x1f,
0x27, 0x5a, 0xd4, 0x40, 0x51, 0xc5, 0xa0, 0xf7, 0xa1, 0x89, 0x6d, 0xd4, 0x55, 0x56, 0xb6, 0xfa,
0x93, 0xfe, 0xe2, 0x40, 0xfb, 0x90, 0x5f, 0x62, 0x1a, 0x19, 0x79, 0x02, 0xab, 0xa1, 0xe2, 0xc9,
0x88, 0xa7, 0x23, 0x54, 0xea, 0xec, 0x7c, 0x58, 0xb5, 0xb0, 0x54, 0xdb, 0x2e, 0x74, 0x0e, 0x12,
0x95, 0x5e, 0xb1, 0xd2, 0xe4, 0xbd, 0x2f, 0xa1, 0x7b, 0x43, 0xa4, 0xe3, 0x9d, 0x89, 0xab, 0xa2,
0xab, 0x67, 0xe2, 0x4a, 0xd7, 0x7f, 0xce, 0xe3, 0xdc, 0xf4, 0xca, 0x63, 0x86, 0xf8, 0xa2, 0xf1,
0x99, 0x43, 0x77, 0x81, 0xec, 0xa7, 0x82, 0x2b, 0x81, 0x41, 0x0e, 0x45, 0x96, 0xf1, 0x53, 0xb1,
0xb8, 0xe3, 0xa6, 0x8b, 0x8d, 0x5a, 0x17, 0xe9, 0x27, 0x40, 0x06, 0x22, 0x16, 0x4a, 0x58, 0xf4,
0xbd, 0xc2, 0x03, 0x0d, 0x8b, 0x68, 0xb7, 0xeb, 0x92, 0x87, 0xe0, 0x69, 0xf0, 0x62, 0xb0, 0xce,
0xce, 0xdd, 0xaa, 0x23, 0x25, 0xca, 0x19, 0x2a, 0xd0, 0xa8, 0x70, 0x6a, 0x01, 0x7f, 0x4b, 0x09,
0x73, 0x40, 0x53, 0x84, 0x72, 0xa7, 0x43, 0x95, 0x2b, 0x64, 0x43, 0xed, 0x16, 0xb5, 0x2e, 0x1b,
0x8a, 0x9e, 0x96, 0xc9, 0xea, 0x9d, 0x58, 0x26, 0xd9, 0x8f, 0xa0, 0x89, 0xb6, 0x36, 0xdb, 0x99,
0x6d, 0x33, 0x52, 0xfa, 0xac, 0x4c, 0x75, 0xd9, 0x40, 0xf7, 0xea, 0x81, 0xda, 0x85, 0xdf, 0x1f,
0xac, 0xae, 0xde, 0x9e, 0x23, 0x6d, 0x63, 0x3c, 0xe1, 0xf7, 0xe2, 0x99, 0x4d, 0x35, 0x52, 0xfb,
0xd6, 0xeb, 0x96, 0xf9, 0x6e, 0xdf, 0xd5, 0xbe, 0x91, 0xa0, 0x8f, 0xa1, 0x15, 0x0e, 0x5f, 0x88,
0x31, 0x27, 0x1f, 0xc3, 0x0a, 0xa6, 0x26, 0x32, 0xbb, 0x11, 0x77, 0xa6, 0xe6, 0xcf, 0x0a, 0x39,
0x1d, 0xd8, 0x92, 0x16, 0x24, 0xd4, 0xc2, 0xd0, 0x99, 0xef, 0xcd, 0xdc, 0x26, 0xcd, 0x67, 0x56,
0x4c, 0x0f, 0xc0, 0x3d, 0x61, 0x81, 0xde, 0x74, 0xcc, 0xa0, 0xf0, 0x62, 0x29, 0xed, 0xfb, 0x6b,
0x99, 0x29, 0xdb, 0x20, 0xfc, 0xd6, 0xbc, 0xef, 0x64, 0xaa, 0xb0, 0x3d, 0x5d, 0x86, 0xdf, 0xf4,
0x47, 0xf0, 0x8e, 0xe4, 0x48, 0x90, 0x35, 0x68, 0x04, 0x03, 0xeb, 0xa3, 0x11, 0x0c, 0xc8, 0x07,
0xe8, 0xde, 0xf6, 0xa5, 0x5b, 0x25, 0x71, 0xc2, 0x02, 0x86, 0x81, 0x1f, 0x40, 0x37, 0xc8, 0xf6,
0xa5, 0x4c, 0x47, 0x51, 0xc2, 0x95, 0x4c, 0xd1, 0xeb, 0x2a, 0xbb, 0xc9, 0xa4, 0xbb, 0xd0, 0xd3,
0xee, 0x43, 0xc5, 0x55, 0x89, 0xbe, 0x0d, 0x68, 0x69, 0x5e, 0x19, 0xce, 0x52, 0xb8, 0xad, 0x5a,
0xaf, 0x18, 0x2a, 0x12, 0xf4, 0x5b, 0xe3, 0xe1, 0xe0, 0x5c, 0x24, 0xaa, 0x06, 0x0a, 0xa4, 0xd1,
0x41, 0x97, 0x19, 0x82, 0x50, 0x53, 0x8a, 0xcd, 0x79, 0xad, 0xca, 0x59, 0x73, 0x19, 0xca, 0xe8,
0x6f, 0x0e, 0x40, 0x91, 0x50, 0x9e, 0x95, 0x26, 0xce, 0x62, 0x13, 0xf2, 0x69, 0xed, 0xf2, 0xcd,
0xe2, 0xa4, 0x14, 0xb1, 0xda, 0x7d, 0xdc, 0x2a, 0x60, 0x61, 0x21, 0xdf, 0xab, 0xf4, 0x0d, 0xdf,
0x8e, 0x49, 0x9f, 0x82, 0xee, 0x7e, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0xa1, 0x0d, 0xa3,
0xec, 0x4f, 0xc5, 0x98, 0xdf, 0x22, 0xf2, 0x00, 0x9a, 0x3a, 0x53, 0x83, 0xcd, 0xd9, 0x32, 0x8c,
0x90, 0x86, 0x76, 0x3b, 0xe6, 0xc2, 0x8e, 0x80, 0x87, 0x6f, 0xad, 0x85, 0x0b, 0x3e, 0xb3, 0x3d,
0x70, 0x0f, 0xa3, 0x04, 0x4b, 0x70, 0x99, 0xfe, 0x44, 0x0e, 0xbf, 0xc4, 0x37, 0x49, 0x73, 0xb8,
0xbe, 0x8f, 0xeb, 0xe6, 0x3a, 0xe8, 0x7d, 0x58, 0x66, 0x67, 0x8b, 0x27, 0xcd, 0xad, 0x3d, 0x69,
0x21, 0xac, 0x9b, 0x4b, 0xf0, 0x26, 0x9d, 0xfe, 0xd9, 0x80, 0x75, 0x26, 0xb2, 0xe8, 0xa5, 0x08,
0x92, 0x4c, 0xa5, 0xf9, 0x50, 0x45, 0x32, 0xd1, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xd5, 0x2e, 0x33,
0xc4, 0xeb, 0x20, 0x89, 0x3c, 0x82, 0xce, 0x34, 0xfa, 0x67, 0x55, 0xeb, 0x2a, 0xe4, 0x11, 0xac,
0x84, 0x32, 0x4f, 0x87, 0xe5, 0x6e, 0x6f, 0x54, 0xda, 0x26, 0x33, 0x23, 0x66, 0x85, 0x5a, 0x0d,
0x47, 0xcd, 0x57, 0xe3, 0x88, 0x3c, 0x99, 0xc2, 0x91, 0xdf, 0x42, 0x83, 0x77, 0x2b, 0x83, 0x1b,
0x62, 0x76, 0x53, 0x9b, 0xfe, 0xea, 0xc0, 0xdb, 0xf5, 0x14, 0x5e, 0x6b, 0x31, 0xca, 0x89, 0x34,
0xe6, 0x4e, 0xc4, 0x9d, 0x37, 0x11, 0xaf, 0x9a, 0x48, 0xf5, 0x3a, 0x37, 0xeb, 0xaf, 0xf3, 0x19,
0xdc, 0x9f, 0x19, 0xd3, 0xbe, 0x1c, 0x4f, 0x34, 0x1e, 0xfe, 0xc7, 0xb8, 0xf4, 0xc9, 0x48, 0x53,
0x3b, 0xa8, 0x36, 0x33, 0x04, 0xfd, 0x1c, 0xde, 0x09, 0x85, 0xaa, 0x0d, 0xa9, 0x40, 0x5b, 0x1f,
0xdc, 0x23, 0x71, 0xb1, 0xa0, 0x7c, 0x2d, 0xa2, 0x5f, 0x81, 0x7f, 0x32, 0x19, 0x71, 0x25, 0x96,
0xb2, 0xde, 0x83, 0xd5, 0x63, 0x39, 0x91, 0xb1, 0x3c, 0xbd, 0xba, 0x65, 0xe5, 0x7d, 0x58, 0x31,
0xf7, 0xd1, 0xfc, 0xb0, 0xb5, 0x59, 0x41, 0xd2, 0xbb, 0x1a, 0xd0, 0x43, 0x1e, 0x0f, 0xf3, 0x58,
0xa7, 0xa1, 0xff, 0xdc, 0xb2, 0xbd, 0xde, 0x5f, 0xd7, 0x9b, 0xce, 0xdf, 0xd7, 0x9b, 0xce, 0x3f,
0xd7, 0x9b, 0xce, 0x1f, 0xff, 0x6e, 0xbe, 0xf5, 0xbc, 0x85, 0xff, 0xdd, 0x8f, 0xff, 0x0b, 0x00,
0x00, 0xff, 0xff, 0xd3, 0x15, 0x68, 0xea, 0x88, 0x0b, 0x00, 0x00,
}

6
internal/private.proto
View file

@ -6,12 +6,10 @@ message IndexMeta {
}
message FrameMeta {
bool InverseEnabled = 2;
string CacheType = 3;
uint32 CacheSize = 4;
string TimeQuantum = 5;
bool RangeEnabled = 6;
repeated Field Fields = 7;
repeated Field Fields = 7;
}
message ImportResponse {
@ -37,13 +35,11 @@ message Cache {
message MaxSlices {
map<string, uint64> Standard = 1;
map<string, uint64> Inverse = 2;
}
message CreateSliceMessage {
string Index = 1;
uint64 Slice = 2;
bool IsInverse = 3;
}
message DeleteIndexMessage {

158
internal/public.pb.go
View file

@ -43,9 +43,9 @@ var _ = math.Inf
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type Row struct {
Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"`
Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"`
Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
}
func (m *Row) Reset() { *m = Row{} }
@ -53,9 +53,9 @@ func (m *Row) String() string { return proto.CompactTextString(m) }
func (*Row) ProtoMessage() {}
func (*Row) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{0} }
func (m *Row) GetBits() []uint64 {
func (m *Row) GetColumns() []uint64 {
if m != nil {
return m.Bits
return m.Columns
}
return nil
}
@ -267,12 +267,12 @@ func (m *AttrMap) GetAttrs() []*Attr {
}
type QueryRequest struct {
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
ExcludeAttrs bool `protobuf:"varint,6,opt,name=ExcludeAttrs,proto3" json:"ExcludeAttrs,omitempty"`
ExcludeBits bool `protobuf:"varint,7,opt,name=ExcludeBits,proto3" json:"ExcludeBits,omitempty"`
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"`
ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"`
}
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
@ -308,16 +308,16 @@ func (m *QueryRequest) GetRemote() bool {
return false
}
func (m *QueryRequest) GetExcludeAttrs() bool {
func (m *QueryRequest) GetExcludeRowAttrs() bool {
if m != nil {
return m.ExcludeAttrs
return m.ExcludeRowAttrs
}
return false
}
func (m *QueryRequest) GetExcludeBits() bool {
func (m *QueryRequest) GetExcludeColumns() bool {
if m != nil {
return m.ExcludeBits
return m.ExcludeColumns
}
return false
}
@ -575,10 +575,10 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.Bits) > 0 {
dAtA2 := make([]byte, len(m.Bits)*10)
if len(m.Columns) > 0 {
dAtA2 := make([]byte, len(m.Columns)*10)
var j1 int
for _, num := range m.Bits {
for _, num := range m.Columns {
for num >= 1<<7 {
dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -902,20 +902,20 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.ExcludeAttrs {
if m.ExcludeRowAttrs {
dAtA[i] = 0x30
i++
if m.ExcludeAttrs {
if m.ExcludeRowAttrs {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if m.ExcludeBits {
if m.ExcludeColumns {
dAtA[i] = 0x38
i++
if m.ExcludeBits {
if m.ExcludeColumns {
dAtA[i] = 1
} else {
dAtA[i] = 0
@ -1263,9 +1263,9 @@ func encodeVarintPublic(dAtA []byte, offset int, v uint64) int {
func (m *Row) Size() (n int) {
var l int
_ = l
if len(m.Bits) > 0 {
if len(m.Columns) > 0 {
l = 0
for _, e := range m.Bits {
for _, e := range m.Columns {
l += sovPublic(uint64(e))
}
n += 1 + sovPublic(uint64(l)) + l
@ -1405,10 +1405,10 @@ func (m *QueryRequest) Size() (n int) {
if m.Remote {
n += 2
}
if m.ExcludeAttrs {
if m.ExcludeRowAttrs {
n += 2
}
if m.ExcludeBits {
if m.ExcludeColumns {
n += 2
}
return n
@ -1615,7 +1615,7 @@ func (m *Row) Unmarshal(dAtA []byte) error {
break
}
}
m.Bits = append(m.Bits, v)
m.Columns = append(m.Columns, v)
} else if wireType == 2 {
var packedLen int
for shift := uint(0); ; shift += 7 {
@ -1655,10 +1655,10 @@ func (m *Row) Unmarshal(dAtA []byte) error {
break
}
}
m.Bits = append(m.Bits, v)
m.Columns = append(m.Columns, v)
}
} else {
return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Columns", wireType)
}
case 2:
if wireType != 2 {
@ -2602,7 +2602,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
m.Remote = bool(v != 0)
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeAttrs", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeRowAttrs", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
@ -2619,10 +2619,10 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.ExcludeAttrs = bool(v != 0)
m.ExcludeRowAttrs = bool(v != 0)
case 7:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeBits", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeColumns", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
@ -2639,7 +2639,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.ExcludeBits = bool(v != 0)
m.ExcludeColumns = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -3795,50 +3795,50 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 707 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a,
0x14, 0xbe, 0x13, 0x3b, 0x7f, 0x27, 0x49, 0x55, 0x8d, 0xee, 0xed, 0xb5, 0xae, 0xae, 0x82, 0x65,
0xb1, 0xf0, 0x2a, 0x95, 0xc2, 0x1e, 0x44, 0xfa, 0x23, 0x45, 0x15, 0x15, 0x4c, 0x4a, 0x59, 0xbb,
0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x08, 0x16,
0x88, 0x15, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x73, 0xc6, 0x13, 0x3b, 0xad, 0x04, 0x2c, 0xd8,
0xcd, 0xf7, 0x7d, 0x33, 0xc7, 0xf3, 0xcd, 0xf9, 0x4e, 0x02, 0xc3, 0xac, 0xbc, 0x48, 0xe2, 0xcb,
0x49, 0x96, 0x2b, 0xad, 0x78, 0x2f, 0x4e, 0xb5, 0xcc, 0xd3, 0x28, 0x09, 0x16, 0xe0, 0x08, 0xb5,
0xe2, 0x1c, 0xdc, 0x59, 0xac, 0x0b, 0x8f, 0xf9, 0x4e, 0xe8, 0x0a, 0x5a, 0xf3, 0x87, 0xd0, 0x7e,
0xaa, 0x75, 0x5e, 0x78, 0x2d, 0xdf, 0x09, 0x07, 0xd3, 0x9d, 0x89, 0x3d, 0x34, 0x41, 0x5a, 0x18,
0x11, 0x4f, 0x9e, 0xc8, 0x75, 0xe1, 0x39, 0xbe, 0x13, 0xf6, 0x05, 0xad, 0x83, 0xc7, 0xe0, 0x3e,
0x8f, 0xe2, 0x9c, 0xef, 0x40, 0x6b, 0x7e, 0xe8, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd, 0x0f, 0xf9,
0xdf, 0xd0, 0x3e, 0x50, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x5d, 0x70, 0x4e, 0xe4, 0xda,
0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0xce, 0xa3, 0x64, 0xa3, 0x9e, 0x47, 0x09,
0x15, 0x71, 0x04, 0x2e, 0xb7, 0xab, 0x38, 0x55, 0x95, 0xe0, 0x25, 0x38, 0xb3, 0x58, 0xa3, 0x28,
0xd4, 0x6a, 0xf3, 0x55, 0x03, 0xf8, 0x7f, 0xd0, 0x3b, 0x50, 0x49, 0xb9, 0x4c, 0xe7, 0x87, 0xd5,
0xb7, 0x37, 0x98, 0xff, 0x0f, 0xfd, 0xb3, 0x78, 0x29, 0x0b, 0x1d, 0x2d, 0x33, 0xba, 0x84, 0x23,
0x6a, 0x22, 0x78, 0x05, 0x23, 0xb3, 0x13, 0xdd, 0x2e, 0xa4, 0xbe, 0xe7, 0xe9, 0xf7, 0x5e, 0xe9,
0xbe, 0xc7, 0xf7, 0x0c, 0x5c, 0xd4, 0xac, 0xc4, 0x36, 0x12, 0x3e, 0xe9, 0xd9, 0x3a, 0x93, 0xd5,
0x4d, 0x69, 0xcd, 0x7d, 0x18, 0x2c, 0x74, 0x1e, 0xa7, 0xd7, 0xe7, 0x51, 0x52, 0xca, 0xaa, 0x50,
0x93, 0x42, 0x8f, 0xf3, 0x54, 0x1b, 0xd9, 0x25, 0x1b, 0x1b, 0x8c, 0x1e, 0x67, 0x4a, 0x25, 0x46,
0x6c, 0xfb, 0x2c, 0xec, 0x89, 0x9a, 0xe0, 0x63, 0x80, 0xe3, 0x44, 0x45, 0xd5, 0xd9, 0x8e, 0xcf,
0x42, 0x26, 0x1a, 0x4c, 0xb0, 0x0f, 0x5d, 0xbc, 0xe9, 0xb3, 0x28, 0xab, 0xdd, 0xb2, 0x9f, 0xb8,
0x0d, 0x3e, 0x30, 0x18, 0xbe, 0x28, 0x65, 0xbe, 0x16, 0xf2, 0x4d, 0x29, 0x0b, 0xea, 0x0a, 0xe1,
0xca, 0xa5, 0x01, 0x7c, 0x0f, 0x3a, 0x8b, 0x24, 0xbe, 0x94, 0xe6, 0xed, 0x5c, 0x51, 0x21, 0xf4,
0x5a, 0xbf, 0x79, 0x41, 0x5e, 0x7b, 0xa2, 0x49, 0xe1, 0x49, 0x21, 0x97, 0x4a, 0x5b, 0x33, 0x15,
0xe2, 0x01, 0x0c, 0x8f, 0x6e, 0x2e, 0x93, 0xf2, 0x4a, 0x9a, 0xa3, 0x1d, 0x52, 0xb7, 0x38, 0xac,
0x5e, 0x61, 0x4a, 0x7c, 0xd7, 0x54, 0x6f, 0x50, 0xc1, 0x5b, 0x06, 0xa3, 0xea, 0xfa, 0x45, 0xa6,
0xd2, 0x42, 0x62, 0x8f, 0x8e, 0xf2, 0xdc, 0xf6, 0xe8, 0x28, 0xcf, 0xf9, 0x3e, 0x74, 0x85, 0x2c,
0xca, 0x44, 0xdb, 0xc6, 0xff, 0x53, 0x3f, 0x85, 0x3d, 0x5b, 0x26, 0x5a, 0xd8, 0x5d, 0xfc, 0x09,
0xec, 0x6c, 0x05, 0xc9, 0x4c, 0xcc, 0x60, 0xfa, 0x6f, 0x7d, 0x6e, 0x4b, 0x17, 0x77, 0xb6, 0x07,
0x1f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x03, 0x9a, 0x5c, 0xba, 0xd3, 0x60, 0x3a, 0xaa, 0xab, 0x08,
0xb5, 0x12, 0x34, 0xd3, 0x43, 0x60, 0xa7, 0x55, 0x86, 0xd8, 0x29, 0x76, 0x0e, 0x67, 0xd2, 0x7e,
0xb6, 0xd1, 0x39, 0xa4, 0x85, 0x11, 0xb9, 0x07, 0xdd, 0x83, 0xd7, 0x51, 0x7a, 0x2d, 0xaf, 0x28,
0x43, 0x3d, 0x61, 0x21, 0x9f, 0xd4, 0x33, 0x49, 0x8f, 0x3e, 0x98, 0xf2, 0xba, 0x84, 0x55, 0x44,
0x3d, 0xb7, 0x36, 0xc4, 0xd8, 0x82, 0x91, 0x09, 0x71, 0xf0, 0x8d, 0xc1, 0x68, 0xbe, 0xcc, 0x54,
0xae, 0x1b, 0xc1, 0x98, 0xa7, 0x57, 0xf2, 0xc6, 0x06, 0x83, 0x00, 0xb2, 0xc7, 0x79, 0xb4, 0x34,
0x13, 0xd0, 0x17, 0x06, 0x20, 0x4b, 0x01, 0xa1, 0x40, 0xb8, 0xc2, 0x00, 0x8a, 0x02, 0xce, 0x78,
0xe1, 0xb9, 0x26, 0x44, 0x06, 0x61, 0xe4, 0xed, 0x88, 0x17, 0x5e, 0x9b, 0xa4, 0x9a, 0xc0, 0xc8,
0x6f, 0x66, 0x1c, 0x63, 0xe2, 0x84, 0x8e, 0x68, 0x30, 0xf8, 0x0e, 0x42, 0xad, 0xe8, 0x87, 0xad,
0x4b, 0x3f, 0x6c, 0x16, 0xe2, 0x49, 0x53, 0x86, 0xc4, 0x1e, 0x89, 0x0d, 0x26, 0xf8, 0xc4, 0x80,
0x1b, 0x8f, 0x34, 0x3c, 0x7f, 0xce, 0x28, 0xee, 0x8d, 0x65, 0x62, 0x1a, 0x83, 0x7b, 0x11, 0xfc,
0xc2, 0xe6, 0x1e, 0x74, 0xe8, 0x16, 0xd6, 0x62, 0x85, 0xee, 0x98, 0xe8, 0xde, 0x35, 0x31, 0xdb,
0xfd, 0x7c, 0x3b, 0x66, 0x5f, 0x6e, 0xc7, 0xec, 0xeb, 0xed, 0x98, 0xbd, 0xfb, 0x3e, 0xfe, 0xeb,
0xa2, 0x43, 0x7f, 0x1c, 0x8f, 0x7e, 0x04, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x35, 0x66, 0x28, 0x48,
0x06, 0x00, 0x00,
// 709 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c,
0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c,
0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba,
0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4,
0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd,
0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64,
0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8,
0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82,
0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69,
0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f,
0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd,
0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e,
0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e,
0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62,
0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf,
0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8,
0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77,
0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e,
0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35,
0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88,
0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59,
0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7,
0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c,
0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd,
0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85,
0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3,
0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18,
0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3,
0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4,
0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96,
0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60,
0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec,
0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd,
0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e,
0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c,
0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41,
0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62,
0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9,
0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b,
0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82,
0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8,
0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd,
0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7,
0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7,
0xa4, 0x5a, 0x06, 0x00, 0x00,
}

6
internal/public.proto
View file

@ -3,7 +3,7 @@ syntax = "proto3";
package internal;
message Row {
repeated uint64 Bits = 1;
repeated uint64 Columns = 1;
repeated string Keys = 3;
repeated Attr Attrs = 2;
}
@ -49,8 +49,8 @@ message QueryRequest {
repeated uint64 Slices = 2;
bool ColumnAttrs = 3;
bool Remote = 5;
bool ExcludeAttrs = 6;
bool ExcludeBits = 7;
bool ExcludeRowAttrs = 6;
bool ExcludeColumns = 7;
}
message QueryResponse {

7
pilosa.go
View file

@ -32,10 +32,9 @@ var (
ErrIndexNotFound = errors.New("index not found")
// ErrFrameRequired is returned when no frame is specified.
ErrFrameRequired = errors.New("frame required")
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
ErrFrameInverseDisabled = errors.New("frame inverse disabled")
ErrFrameRequired = errors.New("frame required")
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")

28
pql/ast.go
View file

@ -177,34 +177,6 @@ func (c *Call) String() string {
return buf.String()
}
// SupportsInverse indicates that the call may be on an inverse frame.
func (c *Call) SupportsInverse() bool {
return c.Name == "Bitmap" || c.Name == "TopN"
}
// IsInverse specifies if the call is for an inverse view.
// Return defaults to false unless absolutely sure of inversion.
func (c *Call) IsInverse(rowLabel, columnLabel string) bool {
if c.SupportsInverse() {
// Top-n has an explicit inverse flag.
if c.Name == "TopN" {
inverse, _ := c.Args["inverse"].(bool)
return inverse
}
// Bitmap calls use the row/column labels to determine whether inverse.
_, rowOK, rowErr := c.UintArg(rowLabel)
_, columnOK, columnErr := c.UintArg(columnLabel)
if rowErr != nil || columnErr != nil {
return false
}
if !rowOK && columnOK {
return true
}
}
return false
}
// HasConditionArg returns true if any arg is a conditional.
func (c *Call) HasConditionArg() bool {
for _, v := range c.Args {

66
pql/ast_test.go
View file

@ -67,69 +67,3 @@ func TestCondition_Value(t *testing.T) {
}
})
}
// Ensure call can be converted into a string.
func TestCall_SupportsInverse(t *testing.T) {
t.Run("Bitmap", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap()`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].SupportsInverse() != true {
t.Fatalf("call should support inverse: %s", q.Calls[0])
}
})
t.Run("Count Bitmap", func(t *testing.T) {
q, err := pql.ParseString(`Count(Bitmap())`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].SupportsInverse() == true {
t.Fatalf("call should not support inverse: %s", q.Calls[0])
}
})
t.Run("Union Bitmaps", func(t *testing.T) {
q, err := pql.ParseString(`Union(Bitmap(), Bitmap())`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].SupportsInverse() == true {
t.Fatalf("call should not support inverse: %s", q.Calls[0])
}
})
}
// Ensure call is correctly determined to be against an inverse view.
func TestCall_IsInverse(t *testing.T) {
t.Run("Bitmap Row", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap(frame="f", row=1)`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("row", "col") != false {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
t.Run("Bitmap Column", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap(frame="f", col=1)`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("row", "col") != true {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
t.Run("Bitmap Column No Label", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap(frame="f", col=1)`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("rowX", "colX") != false {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
t.Run("Count", func(t *testing.T) {
q, err := pql.ParseString(`Count(Bitmap(frame="f", col=1))`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("row", "col") != false {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
}

41
row.go
View file

@ -22,7 +22,8 @@ import (
"github.com/pilosa/pilosa/roaring"
)
// Row represents a set of bits.
// Row is a set of integers (the associated columns), and attributes which are
// arbitrary key/value pairs storing metadata about what the row represents.
type Row struct {
segments []RowSegment
@ -31,9 +32,9 @@ type Row struct {
}
// NewRow returns a new instance of Row.
func NewRow(bits ...uint64) *Row {
func NewRow(columns ...uint64) *Row {
r := &Row{}
for _, i := range bits {
for _, i := range columns {
r.SetBit(i)
}
return r
@ -151,12 +152,12 @@ func (r *Row) Difference(other *Row) *Row {
return &Row{segments: segments}
}
// SetBit sets the i-th bit of the row.
// SetBit sets the i-th column of the row.
func (r *Row) SetBit(i uint64) (changed bool) {
return r.createSegmentIfNotExists(i / SliceWidth).SetBit(i)
}
// ClearBit clears the i-th bit of the row.
// ClearBit clears the i-th column of the row.
func (r *Row) ClearBit(i uint64) (changed bool) {
s := r.segment(i / SliceWidth)
if s == nil {
@ -226,7 +227,7 @@ func (r *Row) DecrementCount(i uint64) {
}
}
// Count returns the number of set bits in the row.
// Count returns the number of columns in the row.
func (r *Row) Count() uint64 {
var n uint64
for i := range r.segments {
@ -238,10 +239,10 @@ func (r *Row) Count() uint64 {
// MarshalJSON returns a JSON-encoded byte slice of r.
func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
Attrs map[string]interface{} `json:"attrs"`
Bits []uint64 `json:"bits"`
Attrs map[string]interface{} `json:"attrs"`
Columns []uint64 `json:"columns"`
}
o.Bits = r.Bits()
o.Columns = r.Columns()
o.Attrs = r.Attrs
if o.Attrs == nil {
@ -251,11 +252,11 @@ func (r *Row) MarshalJSON() ([]byte, error) {
return json.Marshal(&o)
}
// Bits returns the bits in r as a slice of ints.
func (r *Row) Bits() []uint64 {
// Columns returns the columns in r as a slice of ints.
func (r *Row) Columns() []uint64 {
a := make([]uint64, 0, r.Count())
for i := range r.segments {
a = append(a, r.segments[i].Bits()...)
a = append(a, r.segments[i].Columns()...)
}
return a
}
@ -267,8 +268,8 @@ func encodeRow(r *Row) *internal.Row {
}
return &internal.Row{
Bits: r.Bits(),
Attrs: encodeAttrs(r.Attrs),
Columns: r.Columns(),
Attrs: encodeAttrs(r.Attrs),
}
}
@ -280,7 +281,7 @@ func decodeRow(pr *internal.Row) *Row {
r := NewRow()
r.Attrs = decodeAttrs(pr.Attrs)
for _, v := range pr.Bits {
for _, v := range pr.Columns {
r.SetBit(v)
}
return r
@ -372,7 +373,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment {
}
}
// SetBit sets the i-th bit of the row.
// SetBit sets the i-th column of the row.
func (s *RowSegment) SetBit(i uint64) (changed bool) {
s.ensureWritable()
changed, _ = s.data.Add(i)
@ -382,7 +383,7 @@ func (s *RowSegment) SetBit(i uint64) (changed bool) {
return changed
}
// ClearBit clears the i-th bit of the row.
// ClearBit clears the i-th column of the row.
func (s *RowSegment) ClearBit(i uint64) (changed bool) {
s.ensureWritable()
@ -398,8 +399,8 @@ func (s *RowSegment) InvalidateCount() {
s.n = s.data.Count()
}
// Bits returns a list of all bits set in the segment.
func (s *RowSegment) Bits() []uint64 {
// Columns returns a list of all columns set in the segment.
func (s *RowSegment) Columns() []uint64 {
a := make([]uint64, 0, s.Count())
itr := s.data.Iterator()
for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
@ -408,7 +409,7 @@ func (s *RowSegment) Bits() []uint64 {
return a
}
// Count returns the number of set bits in the row.
// Count returns the number of set columns in the row.
func (s *RowSegment) Count() uint64 { return s.n }
// ensureWritable clones the segment if it is pointing to non-writable data.

22
row_test.go
View file

@ -47,7 +47,7 @@ func TestRow_Merge(t *testing.T) {
if cnt := test.r1.Count(); cnt != test.exp {
t.Fatalf("merged count %d is not %d", cnt, test.exp)
}
if length := len(test.r1.Bits()); uint64(length) != test.exp {
if length := len(test.r1.Columns()); uint64(length) != test.exp {
t.Fatalf("merged length %d is not %d", length, test.exp)
}
})
@ -65,15 +65,15 @@ func TestRow_Xor(t *testing.T) {
t.Fatalf("Test 1 Count after xor %d != 3\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Results %v != expected %v\n", res.Columns(), exp)
}
res = r2.Xor(r1)
if res.Count() != 3 {
t.Fatalf("Test 3 Count after xor %d != 3\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 4 Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 4 Results %v != expected %v\n", res.Columns(), exp)
}
}
@ -86,15 +86,15 @@ func TestRow_Union_Segment(t *testing.T) {
if res.Count() != 4 {
t.Fatalf("Test 1 Count after Union %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Columns(), exp)
}
res = r2.Union(r1)
if res.Count() != 4 {
t.Fatalf("Test 3 Count after xor %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Columns(), exp)
}
}
@ -107,7 +107,7 @@ func TestRow_Difference_Segment(t *testing.T) {
if res.Count() != 2 {
t.Fatalf("Test 1 Count after Difference %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Columns(), exp)
}
}

26
server.go
View file

@ -444,11 +444,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
if obj.IsInverse {
idx.SetRemoteMaxInverseSlice(obj.Slice)
} else {
idx.SetRemoteMaxSlice(obj.Slice)
}
idx.SetRemoteMaxSlice(obj.Slice)
case *internal.CreateIndexMessage:
opt := IndexOptions{}
_, err := s.Holder.CreateIndex(obj.Index, opt)
@ -573,7 +569,7 @@ func (s *Server) SendTo(to *Node, pb proto.Message) error {
// where a node fails to receive a Broadcast message, or
// when a new (empty) node needs to get in sync with the
// rest of the cluster, two things are shared via gossip:
// - MaxSlice/MaxInverseSlice by Index
// - MaxSlice by Index
// - Schema
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalStatus() (proto.Message, error) {
@ -629,7 +625,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
return errors.Wrap(err, "applying schema")
}
// Sync maxSlices (standard).
// Sync maxSlices.
oldmaxslices := s.Holder.MaxSlices()
for index, newMax := range ns.MaxSlices.Standard {
localIndex := s.Holder.Index(index)
@ -645,22 +641,6 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
}
}
// Sync maxSlices (inverse).
oldMaxInverseSlices := s.Holder.MaxInverseSlices()
for index, newMaxInverse := range ns.MaxSlices.Inverse {
localIndex := s.Holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.logger.Printf("Local Index not found: %s", index)
continue
}
if newMaxInverse > oldMaxInverseSlices[index] {
oldMaxInverseSlices[index] = newMaxInverse
localIndex.SetRemoteMaxInverseSlice(newMaxInverse)
}
}
return nil
}

6
server/cluster_test.go
View file

@ -464,12 +464,12 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// This is an attempt to ensure there is data on both nodes, but is not guaranteed.
// TODO: Deterministic node IDs would ensure consistent results
setBits := ""
setColumns := ""
for i := 0; i < 20; i++ {
setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth)
setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setBits); err != nil {
if _, err := m0.Query("i", "", setColumns); err != nil {
t.Fatal(err)
}

84
server/server_test.go
View file

@ -26,7 +26,6 @@ import (
"strings"
"testing"
"testing/quick"
"time"
"github.com/BurntSushi/toml"
"github.com/pilosa/pilosa"
@ -69,8 +68,8 @@ func TestMain_Set_Quick(t *testing.T) {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"bits": columnIDs,
"attrs": map[string]interface{}{},
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
@ -92,8 +91,8 @@ func TestMain_Set_Quick(t *testing.T) {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"bits": columnIDs,
"attrs": map[string]interface{}{},
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
@ -132,7 +131,7 @@ func TestMain_SetRowAttrs(t *testing.T) {
t.Fatal(err)
}
// Set bits on different rows in different frames.
// Set columns on different rows in different frames.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil {
@ -157,14 +156,14 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Query row x/1.
if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -175,19 +174,19 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Query rows after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
@ -205,7 +204,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
t.Fatal(err)
}
// Set bits on row.
// Set columns on row.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil {
@ -220,7 +219,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
// Query row.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -231,55 +230,12 @@ func TestMain_SetColumnAttrs(t *testing.T) {
// Query row after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}
// Ensure inverse slices get handled correctly in a multi-node query.
func TestMain_InverseSlices(t *testing.T) {
mains := test.MustRunMainWithCluster(t, 2)
m0 := mains[0]
m1 := mains[1]
// Make sure to use node0 in the cluster.
var m *test.Main
if m0.Server.NodeID < m1.Server.NodeID {
m = m0
} else {
m = m1
}
// Create frames.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal("create frame:", err)
}
// Write data on cluster.
if _, err := m.Query("i", "", fmt.Sprintf(`
SetBit(col=1, frame="f", row=1000)
SetBit(col=1, frame="f", row=2000)
SetBit(col=1, frame="f", row=%d)
`, 1*pilosa.SliceWidth)); err != nil {
t.Fatal("setting bits:", err)
}
time.Sleep(1 * time.Second)
// Query the cluster.
if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != fmt.Sprintf(`{"results":[{"attrs":{},"bits":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
// Ensure program can set bits on one cluster and then restore to a second cluster.
// Ensure program can set columns on one cluster and then restore to a second cluster.
func TestMain_FrameRestore(t *testing.T) {
mains1 := test.MustRunMainWithCluster(t, 2)
m10 := mains1[0]
@ -304,13 +260,13 @@ func TestMain_FrameRestore(t *testing.T) {
SetBit(row=1, frame="f", col=600000)
SetBit(row=1, frame="f", col=800000)
`); err != nil {
t.Fatal("setting bits:", err)
t.Fatal("setting columns:", err)
}
// Query row on first cluster.
if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -348,7 +304,7 @@ func TestMain_FrameRestore(t *testing.T) {
// Query row on second cluster.
if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("2unexpected result: %s", res)
}
}
@ -408,7 +364,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
t.Fatal("create frame:", err)
}
// Set some bits
// Set some columns
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
@ -416,7 +372,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
}
}
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {
t.Fatal("setting bits:", err)
t.Fatal("setting columns:", err)
}
// Calculate caches on the first node
@ -440,7 +396,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
}
}
// SetCommand represents a command to set a bit.
// SetCommand represents a command to set a column.
type SetCommand struct {
ID uint64
Frame string

6
stats_test.go
View file

@ -146,7 +146,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
}
}
func TestStatsCount_SetBitmapAttrs(t *testing.T) {
func TestStatsCount_SetColumnAttrs(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -162,8 +162,8 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) {
frame.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "SetBitmapAttrs" {
t.Errorf("Expected SetBitmapAttrs, Results %s", name)
if name != "SetRowAttrs" {
t.Errorf("Expected SetRowAttrs, Results %s", name)
}
called = true
},

8
test/fragment.go
View file

@ -85,7 +85,7 @@ func (f *Fragment) Reopen() error {
return nil
}
// MustSetBits sets bits on a row. Panic on error.
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
@ -95,8 +95,8 @@ func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
}
}
// MustClearBits clears bits on a row. Panic on error.
func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) {
// MustClearColumns clears columns on a row. Panic on error.
func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.ClearBit(rowID, columnID); err != nil {
panic(err)
@ -126,7 +126,7 @@ func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) {
s.attrs[id] = m
}
// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk.
// GenerateImportFill generates a set of row/col pairs that evenly fill a fragment chunk.
func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
ipct := int(pct * 100)
for i := 0; i < SliceWidth*rowN; i++ {

13
view.go
View file

@ -30,14 +30,13 @@ import (
// View layout modes.
const (
ViewStandard = "standard"
ViewInverse = "inverse"
ViewFieldPrefix = "field_"
)
// IsValidView returns true if name is valid.
func IsValidView(name string) bool {
return name == ViewStandard || name == ViewInverse
return name == ViewStandard
}
// View represents a container for frame data.
@ -252,9 +251,8 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
// Send the create slice message to all nodes.
err := v.broadcaster.SendAsync(
&internal.CreateSliceMessage{
Index: v.index,
Slice: slice,
IsInverse: IsInverseView(v.name),
Index: v.index,
Slice: slice,
})
if err != nil {
return nil, errors.Wrap(err, "sending message")
@ -428,11 +426,6 @@ func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint6
return r, nil
}
// IsInverseView returns true if the view is used for storing an inverted representation.
func IsInverseView(name string) bool {
return strings.HasPrefix(name, ViewInverse)
}
// ViewInfo represents schema information for a view.
type ViewInfo struct {
Name string `json:"name"`

4
view_test.go
View file

@ -72,7 +72,7 @@ func (v *View) Reopen() error {
return v.Open()
}
// MustSetBits sets bits on a row. Panic on error.
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
@ -82,7 +82,7 @@ func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) {
}
}
// MustClearBits clears bits on a row. Panic on error.
// MustClearColumns clears columns on a row. Panic on error.
func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := v.ClearBit(rowID, columnID); err != nil {

3
webui/assets/main.js
View file

@ -581,14 +581,11 @@ function parse_query(query, indexname) {
function parse_options(option_str) {
var int_keys = ["cacheSize"];
var bool_keys = ["inverseEnabled"];
var options = {};
for (var i = 0; i < option_str.length; i++) {
var parts = option_str[i].split('=');
if (int_keys.indexOf(parts[0]) !== -1 ){
options[parts[0]] = Number(parts[1])
} else if (bool_keys.indexOf(parts[0]) !== -1){
options[parts[0]] = (parts[1] == "true")
} else {
options[parts[0]] = parts[1]
}