scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-10-09 12:27:52 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into update-tls-config

Browse source
This commit is contained in:
Yuce Tekol 2018-05-29 19:01:51 +03:00 • committed by GitHub
commit 998ebf43f9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
55 changed files with 514 additions and 2539 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.travis.yml
View file

@ -16,10 +16,6 @@ install:
- make install-dep install-statik vendor generate-statik
script:
- make test
# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009
after_success:
- go get github.com/mattn/goveralls
- $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go"
before_deploy:
- pip install awscli --user `whoami`
deploy:
@ -33,6 +29,7 @@ deploy:
matrix:
allow_failures:
- go: master
fast_finish: true
notifications:
slack:
secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="

96
api.go
View file

@ -99,9 +99,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
return resp, errors.Wrap(err, "parsing")
}
execOpts := &ExecOptions{
Remote: req.Remote,
ExcludeAttrs: req.ExcludeAttrs,
ExcludeBits: req.ExcludeBits,
Remote: req.Remote,
ExcludeRowAttrs: req.ExcludeRowAttrs,
ExcludeColumns: req.ExcludeColumns,
}
results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts)
if err != nil {
@ -110,7 +110,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
resp.Results = results
// Fill column attributes if requested.
if req.ColumnAttrs && !req.ExcludeBits {
if req.ColumnAttrs && !req.ExcludeColumns {
// Consolidate all column ids across all calls.
var columnIDs []uint64
for _, result := range results {
@ -118,7 +118,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er
if !ok {
continue
}
columnIDs = uint64Slice(columnIDs).merge(bm.Bits())
columnIDs = uint64Slice(columnIDs).merge(bm.Columns())
}
// Retrieve column attributes across all calls.
@ -305,7 +305,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin
// Wrap writer with a CSV writer.
cw := csv.NewWriter(w)
// Iterate over each bit.
// Iterate over each column.
if err := f.ForEachBit(func(rowID, columnID uint64) error {
return cw.Write([]string{
strconv.FormatUint(rowID, 10),
@ -431,79 +431,6 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName
return blocks, nil
}
// RestoreFrame reads all the data that this host should have for a given frame
// from replicas in the cluster and restores that data to it.
func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error {
if err := api.validate(apiRestoreFrame); err != nil {
return errors.Wrap(err, "validating api method")
}
// Create a client for the remote cluster.
client := NewInternalHTTPClientFromURI(host, api.RemoteClient)
// Determine the maximum number of slices.
maxSlices, err := client.MaxSliceByIndex(ctx)
if err != nil {
return errors.Wrap(err, "getting max slice")
}
// Retrieve frame.
f := api.Holder.Frame(indexName, frameName)
if f == nil {
return ErrFrameNotFound
}
// Retrieve list of all views.
views, err := client.FrameViews(ctx, indexName, frameName)
if err != nil {
return errors.Wrap(err, "getting views")
}
// Loop over each slice and import it if this node owns it.
for slice := uint64(0); slice <= maxSlices[indexName]; slice++ {
// Ignore this slice if we don't own it.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
continue
}
// Loop over view names.
for _, view := range views {
// Create view.
v, err := f.CreateViewIfNotExists(view)
if err != nil {
return errors.Wrap(err, "creating view")
}
// Otherwise retrieve the local fragment.
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return errors.Wrap(err, "creating fragment")
}
// Stream backup from remote node.
rd, err := client.BackupSlice(ctx, indexName, frameName, view, slice)
if err != nil {
return errors.Wrap(err, "getting backup")
} else if rd == nil {
continue // slice doesn't exist
}
// Restore to local frame and always close reader.
if err := func() error {
defer rd.Close()
if _, err := frag.ReadFrom(rd); err != nil {
return errors.Wrap(err, "reading fragment")
}
return nil
}(); err != nil {
return err
}
}
}
return nil
}
// Hosts returns a list of the hosts in the cluster including their ID,
// URL, and which is the coordinator.
func (api *API) Hosts(ctx context.Context) []*Node {
@ -784,7 +711,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error {
// Import into fragment.
err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -803,7 +730,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest
// Import into fragment.
err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values)
if err != nil {
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err)
}
return errors.Wrap(err, "importing")
}
@ -813,12 +740,6 @@ func (api *API) MaxSlices(ctx context.Context) map[string]uint64 {
return api.Holder.MaxSlices()
}
// MaxInverseSlices returns the maximum inverse slice number for each index in a
// map.
func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 {
return api.Holder.MaxInverseSlices()
}
// StatsWithTags returns an instance of whatever implementation of StatsClient
// pilosa is using with the given tags.
func (api *API) StatsWithTags(tags []string) StatsClient {
@ -968,7 +889,6 @@ const (
//apiLocalID // not implemented
//apiLongQueryTime // not implemented
apiMarshalFragment
//apiMaxInverseSlices // not implemented
//apiMaxSlices // not implemented
apiQuery
apiRecalculateCaches

4
cache.go
View file

@ -168,7 +168,7 @@ func NewRankCache(maxEntries uint32) *RankCache {
func (c *RankCache) Add(id uint64, n uint64) {
c.mu.Lock()
defer c.mu.Unlock()
// Ignore if the bit count is below the threshold.
// Ignore if the column count is below the threshold.
if n < c.thresholdValue {
return
}
@ -469,7 +469,7 @@ type BitmapCache interface {
// SimpleCache implements BitmapCache
// it is meant to be a short-lived cache for cases where writes are continuing to access
// the same bit within a short time frame (i.e. good for write-heavy loads)
// the same row within a short time frame (i.e. good for write-heavy loads)
// A read-heavy use case would cause the cache to get bigger, potentially causing the
// node to run out of memory.
type SimpleCache struct {

223
client.go
View file

@ -15,20 +15,17 @@
package pilosa
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"net/http"
"net/url"
"sort"
"strconv"
"time"
"crypto/tls"
@ -77,16 +74,11 @@ func (c *InternalHTTPClient) Host() *URI { return c.defaultURI }
// MaxSliceByIndex returns the number of slices on a server by index.
func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByIndex(ctx, false)
}
// MaxInverseSliceByIndex returns the number of inverse slices on a server by index.
func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByIndex(ctx, true)
return c.maxSliceByIndex(ctx)
}
// maxSliceByIndex returns the number of slices on a server by index.
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
// Execute request against the host.
u := uriPathToURL(c.defaultURI, "/slices/max")
@ -112,9 +104,6 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool)
return nil, fmt.Errorf("json decode: %s", err)
}
if inverse {
return rsp.Inverse, nil
}
return rsp.Standard, nil
}
@ -308,15 +297,15 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl
return nil
}
// ImportK bulk imports bits to a host.
func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, bits []Bit) error {
// ImportK bulk imports bits specified by string keys to a host.
func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
buf, err := marshalImportPayloadK(index, frame, bits)
buf, err := marshalImportPayloadK(index, frame, columns)
if err != nil {
return fmt.Errorf("Error Creating Payload: %s", err)
}
@ -356,7 +345,7 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte
columnIDs := Bits(bits).ColumnIDs()
timestamps := Bits(bits).Timestamps()
// Marshal bits to protobufs.
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
Index: index,
Frame: frame,
@ -378,7 +367,7 @@ func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) {
columnKeys := Bits(bits).ColumnKeys()
timestamps := Bits(bits).Timestamps()
// Marshal bits to protobufs.
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportRequest{
Index: index,
Frame: frame,
@ -465,7 +454,7 @@ func marshalImportValuePayload(index, frame, field string, slice uint64, vals []
columnIDs := FieldValues(vals).ColumnIDs()
values := FieldValues(vals).Values()
// Marshal bits to protobufs.
// Marshal data to protobuf.
buf, err := proto.Marshal(&internal.ImportValueRequest{
Index: index,
Frame: frame,
@ -524,7 +513,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
} else if !(view == ViewStandard || view == ViewInverse) {
} else if view != ViewStandard {
return ErrInvalidView
}
@ -589,109 +578,6 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
return nil
}
// BackupTo backs up an entire frame from a cluster to w.
func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
// Create tar writer around writer.
tw := tar.NewWriter(w)
// Find the maximum number of slices.
var maxSlices map[string]uint64
var err error
if view == ViewStandard {
maxSlices, err = c.MaxSliceByIndex(ctx)
} else if view == ViewInverse {
maxSlices, err = c.MaxInverseSliceByIndex(ctx)
} else {
return ErrInvalidView
}
if err != nil {
return fmt.Errorf("slice n: %s", err)
}
// Backup every slice to the tar file.
for i := uint64(0); i <= maxSlices[index]; i++ {
if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil {
return errors.Wrap(err, "backing up slice")
}
}
// Close tar file.
if err := tw.Close(); err != nil {
return errors.Wrap(err, "closing")
}
return nil
}
// backupSliceTo backs up a single slice to tw.
func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error {
// Return error if unable to backup from any slice.
r, err := c.BackupSlice(ctx, index, frame, view, slice)
if err != nil {
return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err)
} else if r == nil {
return nil
}
defer r.Close()
// Read entire buffer to determine file size.
data, err := ioutil.ReadAll(r)
if err != nil {
return errors.Wrap(err, "reading")
} else if err := r.Close(); err != nil {
return errors.Wrap(err, "closing")
}
// Write slice file header.
if err := tw.WriteHeader(&tar.Header{
Name: strconv.FormatUint(slice, 10),
Mode: 0666,
Size: int64(len(data)),
ModTime: time.Now(),
}); err != nil {
return errors.Wrap(err, "writing header")
}
// Write buffer to file.
if _, err := tw.Write(data); err != nil {
return errors.Wrap(err, "writing buffer")
}
return nil
}
// BackupSlice retrieves a streaming backup from a single slice.
// This function tries slice owners until one succeeds.
func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) {
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, index, slice)
if err != nil {
return nil, fmt.Errorf("slice nodes: %s", err)
}
// Try to backup slice from each one until successful.
for _, i := range rand.Perm(len(nodes)) {
r, err := c.backupSliceNode(ctx, index, frame, view, slice, nodes[i])
if err == nil {
return r, nil // successfully attached
} else if err == ErrFragmentNotFound {
return nil, nil // slice doesn't exist
} else if err != nil {
log.Println(err)
continue
}
}
return nil, fmt.Errorf("unable to connect to any owner")
}
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) {
node := &Node{
URI: uri,
@ -734,86 +620,6 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
return resp.Body, nil
}
// RestoreFrom restores a frame from a backup file to an entire cluster.
func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
// Create tar reader around input.
tr := tar.NewReader(r)
// Process each file.
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
} else if err != nil {
return errors.Wrap(err, "opening")
}
// Parse slice from entry name.
slice, err := strconv.ParseUint(hdr.Name, 10, 64)
if err != nil {
return fmt.Errorf("invalid backup entry: %s", hdr.Name)
}
// Read file into buffer.
var buf bytes.Buffer
if _, err := io.CopyN(&buf, tr, hdr.Size); err != nil {
return errors.Wrap(err, "copying")
}
// Restore file to all nodes that own it.
if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil {
return errors.Wrap(err, "restoring")
}
}
}
// restoreSliceFrom restores a single slice to all owning nodes.
func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error {
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, index, slice)
if err != nil {
return fmt.Errorf("slice nodes: %s", err)
}
// Restore slice to each owner.
for _, node := range nodes {
u := nodePathToURL(node, "/fragment/data")
u.RawQuery = url.Values{
"index": {index},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode()
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("User-Agent", "pilosa/"+Version)
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
resp.Body.Close()
// Return error if response not OK.
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode)
}
}
return nil
}
// CreateFrame creates a new frame on the server.
func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error {
if index == "" {
@ -1140,7 +946,8 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto
return nil
}
// Bit represents the location of a single bit.
// Bit represents the intersection of a row and a column. It can be specifed by
// integer ids or string keys.
type Bit struct {
RowID uint64
ColumnID uint64
@ -1149,7 +956,7 @@ type Bit struct {
Timestamp int64
}
// Bits represents a slice of bits.
// Bits is a slice of Bit.
type Bits []Bit
func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
@ -1277,7 +1084,7 @@ func (p FieldValues) GroupBySlice() map[uint64][]FieldValue {
return m
}
// BitsByPos represents a slice of bits sorted by internal position.
// BitsByPos is a slice of bits sorted row then column.
type BitsByPos []Bit
func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
@ -1314,7 +1121,6 @@ func nodePathToURL(node *Node, path string) url.URL {
// I don't want to let it go unquestioned.
type InternalClient interface {
MaxSliceByIndex(ctx context.Context) (map[string]uint64, error)
MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error)
Schema(ctx context.Context) ([]*IndexInfo, error)
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error)
@ -1326,9 +1132,6 @@ type InternalClient interface {
EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error
ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error
ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error
BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error
BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error)
RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error
CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error
RestoreFrame(ctx context.Context, host, index, frame string) error
FrameViews(ctx context.Context, index, frame string) ([]string, error)

190
client_test.go
View file

@ -15,7 +15,6 @@
package pilosa_test
import (
"bytes"
"context"
"fmt"
"net/http"
@ -232,65 +231,11 @@ func TestClient_Import(t *testing.T) {
}
// Verify data.
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) {
t.Fatalf("unexpected bits: %+v", a)
}
}
// Ensure client can bulk import data to an inverse frame.
func TestClient_ImportInverseEnabled(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frameOpts := pilosa.FrameOptions{
InverseEnabled: true,
}
frame, err := idx.CreateFrameIfNotExists("f", frameOpts)
if err != nil {
panic(err)
}
v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse)
if err != nil {
panic(err)
}
f, err := v.CreateFragmentIfNotExists(0)
if err != nil {
panic(err)
}
// Load bitmap into cache to ensure cache gets updated.
f.Row(0)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
{RowID: 200, ColumnID: 5},
{RowID: 200, ColumnID: 6},
}); err != nil {
t.Fatal(err)
}
// Verify data.
if a := f.Row(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected bits: %+v", a)
}
if a := f.Row(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) {
t.Fatalf("unexpected bits: %+v", a)
}
if a := f.Row(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
@ -370,133 +315,6 @@ func TestClient_ImportValue(t *testing.T) {
}
}
// Ensure client backup and restore a frame.
func TestClient_BackupRestore(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
// Backup from frame.
var buf bytes.Buffer
if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewStandard); err != nil {
t.Fatal(err)
}
// Restore to a different frame.
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil {
t.Fatal(err)
}
// Verify data.
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) {
t.Fatalf("unexpected bits: %+v", a)
}
}
// Ensure client backup and restore a frame with inverse view.
func TestClient_BackupInverseView(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frameOpts := pilosa.FrameOptions{
InverseEnabled: true,
}
frame, err := idx.CreateFrameIfNotExists("f", frameOpts)
if err != nil {
panic(err)
}
v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse)
if err != nil {
panic(err)
}
f, err := v.CreateFragmentIfNotExists(0)
if err != nil {
panic(err)
}
f.SetBit(100, 1)
f.SetBit(100, 2)
f.SetBit(100, 3)
f.SetBit(100, SliceWidth-1)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
// Backup from frame.
var buf bytes.Buffer
if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewInverse); err != nil {
t.Fatal(err)
}
// Restore to a different frame.
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
}
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewInverse); err != nil {
t.Fatal(err)
}
// Verify data.
if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected bits(0): %+v", a)
}
}
// backup returns error with invalid view
func TestClient_BackupInvalidView(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
// Backup from frame.
var buf bytes.Buffer
err := c.BackupTo(context.Background(), &buf, "i", "f", "invalid_view")
if err != pilosa.ErrInvalidView {
t.Fatal(err)
}
}
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
func TestClient_FragmentBlocks(t *testing.T) {
hldr := test.MustOpenHolder()

13
cluster.go
View file

@ -626,21 +626,14 @@ func (a viewsByFrame) addView(frame, view string) {
func (c *Cluster) fragsByHost(idx *Index) fragsByHost {
// frameViews is a map of frame to slice of views.
frameViews := make(viewsByFrame)
inverseFrameViews := make(viewsByFrame)
for _, frame := range idx.Frames() {
for _, view := range frame.Views() {
if IsInverseView(view.Name()) {
inverseFrameViews.addView(frame.Name(), view.Name())
} else {
frameViews.addView(frame.Name(), view.Name())
}
frameViews.addView(frame.Name(), view.Name())
}
}
std := c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews)
inv := c.fragCombos(idx.Name(), idx.MaxInverseSlice(), inverseFrameViews)
return std.add(inv)
return c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews)
}
// fragCombos returns a map (by uri) of lists of fragments for a given index

2
cluster_test.go
View file

@ -424,8 +424,6 @@ func TestCluster_ResizeStates(t *testing.T) {
// Add Field Data to node0.
if err := tc.CreateFrame("i", "fields", FrameOptions{
InverseEnabled: false,
//CacheType: CacheTypeNone,
Fields: []*Field{
{
Name: "fld0",

56
cmd/backup.go
View file

@ -1,56 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"io"
"os"
"github.com/pilosa/pilosa/ctl"
"github.com/spf13/cobra"
)
var Backuper *ctl.BackupCommand
func NewBackupCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Backuper = ctl.NewBackupCommand(os.Stdin, os.Stdout, os.Stderr)
backupCmd := &cobra.Command{
Use: "backup",
Short: "Backup data from pilosa.",
Long: `
Backs up the view from across the cluster into a single file.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := Backuper.Run(context.Background()); err != nil {
return err
}
return nil
},
}
flags := backupCmd.Flags()
flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup.")
flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup.")
flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup.")
flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout")
ctl.SetTLSConfig(flags, &Backuper.TLS.CertificatePath, &Backuper.TLS.CertificateKeyPath, &Backuper.TLS.SkipVerify)
return backupCmd
}
func init() {
subcommandFns["backup"] = NewBackupCmd
}

53
cmd/backup_test.go
View file

@ -1,53 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/cmd"
)
func TestBackupHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "backup", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "pilosa backup") || err != nil {
t.Fatalf("Command 'backup --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestBackupConfig(t *testing.T) {
tests := []commandTest{
{
args: []string{"backup", "--output-file", "/somefile"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Backuper.Host, "localhost:12345")
v.Check(cmd.Backuper.Index, "myindex")
v.Check(cmd.Backuper.Frame, "f1")
v.Check(cmd.Backuper.Path, "/somefile")
return v.Error()
},
},
}
executeDry(t, tests)
}

1
cmd/export.go
View file

@ -53,7 +53,6 @@ The file does not contain any headers.
flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export")
flags.StringVarP(&Exporter.View, "view", "v", "standard", "View to export - default standard")
flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify)

8
cmd/export_test.go
View file

@ -44,7 +44,6 @@ frame = "f1"
v.Check(cmd.Exporter.Host, "localhost:12345")
v.Check(cmd.Exporter.Index, "myindex")
v.Check(cmd.Exporter.Frame, "f1")
v.Check(cmd.Exporter.View, "standard")
v.Check(cmd.Exporter.Path, "/somefile")
return v.Error()
},
@ -52,10 +51,3 @@ frame = "f1"
}
executeDry(t, tests)
}
func TestExportInvalidView(t *testing.T) {
output, err := ExecNewRootCommand(t, "export", "-i", "foo", "-f", "bar", "-v", "test")
if !strings.Contains(err.Error(), "invalid view") {
t.Fatalf("Command 'export' with invalid view should error but: err: '%v', output: '%v'", err, output)
}
}

3
cmd/import.go
View file

@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command
importCmd := &cobra.Command{
Use: "import",
Short: "Bulk load data into pilosa.",
Long: `Bulk imports one or more CSV files to a host's index and frame. The bits
Long: `Bulk imports one or more CSV files to a host's index and frame. The data
of the CSV file are grouped by slice for the most efficient import.
The format of the CSV file is:
@ -61,7 +61,6 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.")
flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame")
flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.")
flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked")
flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame")
ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify)

58
cmd/restore.go
View file

@ -1,58 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"io"
"os"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/ctl"
)
var Restorer *ctl.RestoreCommand
func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Restorer = ctl.NewRestoreCommand(os.Stdin, os.Stdout, os.Stderr)
restoreCmd := &cobra.Command{
Use: "restore",
Short: "Restore data to pilosa from a backup file.",
Long: `
Restores a view to the cluster from a backup file.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := Restorer.Run(context.Background()); err != nil {
return err
}
return nil
},
}
flags := restoreCmd.Flags()
flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Restorer.Index, "index", "i", "", "Pilosa index to restore into.")
flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.")
flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.")
flags.StringVarP(&Restorer.Path, "input-file", "d", "", "File to restore data from.")
ctl.SetTLSConfig(flags, &Restorer.TLS.CertificatePath, &Restorer.TLS.CertificateKeyPath, &Restorer.TLS.SkipVerify)
return restoreCmd
}
func init() {
subcommandFns["restore"] = NewRestoreCommand
}

53
cmd/restore_test.go
View file

@ -1,53 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/cmd"
)
func TestRestoreHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "restore", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "pilosa restore") || err != nil {
t.Fatalf("Command 'restore --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestRestoreConfig(t *testing.T) {
tests := []commandTest{
{
args: []string{"restore", "--input-file", "/somefile"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Restorer.Host, "localhost:12345")
v.Check(cmd.Restorer.Index, "myindex")
v.Check(cmd.Restorer.Frame, "f1")
v.Check(cmd.Restorer.Path, "/somefile")
return v.Error()
},
},
}
executeDry(t, tests)
}

94
ctl/backup.go
View file

@ -1,94 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"context"
"io"
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// BackupCommand represents a command for backing up a view.
type BackupCommand struct {
// Destination host and port.
Host string
// Name of the index, frame, view to backup.
Index string
Frame string
View string
// Output file to write to.
Path string
// Standard input/output
*pilosa.CmdIO
TLS server.TLSConfig
}
// NewBackupCommand returns a new instance of BackupCommand.
func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand {
return &BackupCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
}
}
// Run executes the backup.
func (cmd *BackupCommand) Run(ctx context.Context) error {
// Validate arguments.
if cmd.Path == "" {
return errors.New("output file required")
}
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return errors.Wrap(err, "creating client")
}
// Open output file.
f, err := os.Create(cmd.Path)
if err != nil {
return errors.Wrap(err, "creating file")
}
defer f.Close()
// Begin streaming backup.
if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
return errors.Wrap(err, "backing up")
}
// Sync & close file to ensure durability.
if err := f.Sync(); err != nil {
return errors.Wrap(err, "syncing")
} else if err = f.Close(); err != nil {
return errors.Wrap(err, "closing file")
}
return nil
}
func (cmd *BackupCommand) TLSHost() string {
return cmd.Host
}
func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

65
ctl/backup_test.go
View file

@ -1,65 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"bytes"
"context"
"io/ioutil"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
func TestBackupCommand_FileRequired(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewBackupCommand(stdin, stdout, stderr)
err := cm.Run(context.Background())
if err.Error() != "output file required" {
t.Fatalf("expect error: output file required, actual: %s", err)
}
}
func TestBackupCommand_Run(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm := NewBackupCommand(stdin, stdout, stderr)
file, err := ioutil.TempFile("", "import.csv")
cm.Index = "i"
cm.Host = s.Host()
cm.Frame = "f"
cm.View = pilosa.ViewStandard
cm.Path = file.Name()
err = cm.Run(context.Background())
if err != nil {
t.Fatalf("Command not working, error: '%s'", err)
}
}

10
ctl/bench_test.go
View file

@ -15,6 +15,7 @@
package ctl
import (
"bufio"
"bytes"
"context"
"fmt"
@ -86,3 +87,12 @@ func TestBenchCommand_Run(t *testing.T) {
fmt.Println(buf.String())
}
}
// declare stdin, stdout, stderr
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
rder := []byte{}
stdin := bytes.NewReader(rder)
stdout := bufio.NewWriter(&buf)
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}

14
ctl/export.go
View file

@ -33,7 +33,7 @@ type ExportCommand struct {
// Name of the index & frame to export from.
Index string
Frame string
View string
// Filename to export to.
Path string
@ -59,8 +59,6 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
} else if !(cmd.View == pilosa.ViewStandard || cmd.View == pilosa.ViewInverse) {
return pilosa.ErrInvalidView
}
// Use output file, if specified.
@ -83,13 +81,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
}
// Determine slice count.
var maxSlices map[string]uint64
if cmd.View == pilosa.ViewStandard {
maxSlices, err = client.MaxSliceByIndex(ctx)
} else if cmd.View == pilosa.ViewInverse {
maxSlices, err = client.MaxInverseSliceByIndex(ctx)
}
maxSlices, err := client.MaxSliceByIndex(ctx)
if err != nil {
return errors.Wrap(err, "getting slice count")
}
@ -97,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil {
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, pilosa.ViewStandard, slice, w); err != nil {
return errors.Wrap(err, "exporting")
}
}

8
ctl/export_test.go
View file

@ -41,13 +41,6 @@ func TestExportCommand_Validation(t *testing.T) {
if err != pilosa.ErrFrameRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err)
}
cm.Frame = "f"
cm.View = "test"
err = cm.Run(context.Background())
if err != pilosa.ErrInvalidView {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrInvalidView, err)
}
}
func TestExportCommand_Run(t *testing.T) {
@ -70,7 +63,6 @@ func TestExportCommand_Run(t *testing.T) {
cm.Index = "i"
cm.Frame = "f"
cm.View = pilosa.ViewStandard
if err := cm.Run(context.Background()); err != nil {
t.Fatalf("Export Run doesn't work: %s", err)
}

9
ctl/import.go
View file

@ -107,7 +107,6 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
// Import each path and import by slice.
for _, path := range cmd.Paths {
// Parse path into bits.
logger.Printf("parsing: %s", path)
if err := cmd.importPath(ctx, path); err != nil {
return err
@ -236,13 +235,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err
bitsBySlice := pilosa.Bits(bits).GroupBySlice()
// Parse path into bits.
for slice, bits := range bitsBySlice {
for slice, chunk := range bitsBySlice {
if cmd.Sort {
sort.Sort(pilosa.BitsByPos(bits))
sort.Sort(pilosa.BitsByPos(chunk))
}
logger.Printf("importing slice: %d, n=%d", slice, len(bits))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil {
logger.Printf("importing slice: %d, n=%d", slice, len(chunk))
if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil {
return errors.Wrap(err, "importing")
}
}

87
ctl/restore.go
View file

@ -1,87 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"context"
"io"
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// RestoreCommand represents a command for restoring a frame from a backup.
type RestoreCommand struct {
// Destination host and port.
Host string
// Name of the index & frame to backup.
Index string
Frame string
View string
// Import file to read from.
Path string
// Standard input/output
*pilosa.CmdIO
TLS server.TLSConfig
}
// NewRestoreCommand returns a new instance of RestoreCommand.
func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
return &RestoreCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
}
}
// Run executes the restore command.
func (cmd *RestoreCommand) Run(ctx context.Context) error {
// Validate arguments.
if cmd.Path == "" {
return errors.New("backup file required")
}
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return errors.Wrap(err, "creating client")
}
// Open backup file.
f, err := os.Open(cmd.Path)
if err != nil {
return errors.Wrap(err, "opening file")
}
defer f.Close()
// Restore backup file to the cluster.
if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
return errors.Wrap(err, "restoring")
}
return nil
}
func (cmd *RestoreCommand) TLSHost() string {
return cmd.Host
}
func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

75
ctl/restore_test.go
View file

@ -1,75 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"bufio"
"bytes"
"context"
"io"
"io/ioutil"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
func TestRestoreCommand_FileRequired(t *testing.T) {
cm := RestoreCommand{}
ctx := context.Background()
err := cm.Run(ctx)
if err.Error() != "backup file required" {
t.Fatalf("expect error: output file required, actual: '%s'", err)
}
}
func TestRestoreCommand_Run(t *testing.T) {
file, err := ioutil.TempFile("", "restore.csv")
if err != nil {
t.Fatal(err)
}
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm := NewRestoreCommand(stdin, stdout, stderr)
cm.Path = file.Name()
cm.Index = "i"
cm.Frame = "f"
cm.View = pilosa.ViewStandard
cm.Host = s.Host()
cm.Run(context.Background())
if err != nil {
t.Fatalf("Backup Run doesn't work: %s", err)
}
}
// declare stdin, stdout, stderr
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
rder := []byte{}
stdin := bytes.NewReader(rder)
stdout := bufio.NewWriter(&buf)
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}

2
docs/administration.md
View file

@ -65,7 +65,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count
```
<div class="note">
<p>Note that you must first create a frame with range-encoding enabled and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
<p>Note that you must first create a frame and a field. View <a href="../api-reference/#create-frame">Create Frame</a> for more details.</p>
</div>
#### Exporting

1
docs/api-reference.md
View file

@ -105,7 +105,6 @@ The request payload is in JSON, and may contain the `options` field. The `option
* `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame.
* `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`.
* `cacheSize` (int): Number of rows to keep in the cache. Default 50,000.
* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields.
* `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding).
Each individual `field` contains the following:

10
docs/installation.md
View file

@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -58,7 +57,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:
@ -110,7 +108,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -119,7 +116,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:
@ -177,7 +173,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -186,7 +181,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:
@ -268,7 +262,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -277,7 +270,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:
@ -335,7 +327,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -344,7 +335,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:

157
executor.go
View file

@ -80,36 +80,21 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// Don't bother calculating slices for query types that don't require it.
needsSlices := needsSlices(q.Calls)
// MaxSlice can differ between inverse and standard views, so we need
// to send queries to different slices based on orientation.
var inverseSlices []uint64
// If slices are specified, then use that value for slices or
// inverseSlices. If slices aren't specified, then include all of them.
if len(slices) > 0 {
// For inverse queries, the values of `slices` provided to the Execute() method
// on the remote node actually represents inverseSlices.
inverseSlices = slices
} else if needsSlices {
// If slices are specified, then use that value for slices. If slices aren't
// specified, then include all of them.
if len(slices) == 0 && needsSlices {
// Round up the number of slices.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
maxSlice := idx.MaxSlice()
maxInverseSlice := idx.MaxInverseSlice()
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
for i := range slices {
slices[i] = uint64(i)
}
// Generate a slices of all inverse slices.
inverseSlices = make([]uint64, maxInverseSlice+1)
for i := range inverseSlices {
inverseSlices[i] = uint64(i)
}
}
// Optimize handling for bulk attribute insertion.
@ -120,23 +105,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
if call.SupportsInverse() && needsSlices {
// Fetch frame & row label based on argument.
frame := call.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
f := e.Holder.Frame(index, frame)
if f == nil {
return nil, ErrFrameNotFound
}
// If this call is to an inverse frame send to a different list of slices.
if call.IsInverse(rowLabel, columnLabel) {
slices = inverseSlices
}
}
v, err := e.executeCall(ctx, index, call, slices, opt)
if err != nil {
return nil, err
@ -336,7 +304,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
// If the row label is used then return bitmap attributes.
row, _ := other.(*Row)
if c.Name == "Bitmap" {
if opt.ExcludeAttrs {
if opt.ExcludeRowAttrs {
row.Attrs = map[string]interface{}{}
} else {
idx := e.Holder.Index(index)
@ -367,7 +335,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C
}
}
if opt.ExcludeBits {
if opt.ExcludeColumns {
row.segments = []RowSegment{}
}
@ -580,7 +548,6 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
frame, _ := c.Args["frame"].(string)
inverse, _ := c.Args["inverse"].(bool)
n, _, err := c.UintArg("n")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
@ -619,9 +586,6 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
// Determine view.
view := ViewStandard
if inverse {
view = ViewInverse
}
f := e.Holder.Fragment(index, frame, view, slice)
if f == nil {
@ -685,32 +649,19 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
return nil, ErrFrameNotFound
}
// Return an error if both the row and column label are specified.
rowID, rowOK, rowErr := c.UintArg(rowLabel)
columnID, columnOK, columnErr := c.UintArg(columnLabel)
if rowErr != nil || columnErr != nil {
return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr)
if rowErr != nil {
return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr)
}
if rowOK && columnOK {
return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel)
} else if !rowOK && !columnOK {
return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", rowLabel, columnLabel)
if !rowOK {
return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel)
}
// Determine row or column orientation.
view, id := ViewStandard, rowID
if columnOK {
view, id = ViewInverse, columnID
if !f.InverseEnabled() {
return nil, fmt.Errorf("Bitmap() cannot retrieve columns unless inverse storage enabled")
}
}
frag := e.Holder.Fragment(index, frame, view, slice)
frag := e.Holder.Fragment(index, frame, ViewStandard, slice)
if frag == nil {
return NewRow(), nil
}
return frag.Row(id), nil
return frag.Row(rowID), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
@ -761,26 +712,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Read row & column id.
columnID, columnOK, err := c.UintArg(columnLabel)
if err != nil {
return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err)
}
rowID, rowOK, err := c.UintArg(rowLabel)
if err != nil {
return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err)
}
// Determine view.
var id uint64
var viewName string
if columnOK && rowOK {
return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, rowLabel)
} else if !columnOK && !rowOK {
return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, rowLabel)
} else if columnOK {
viewName, id = ViewInverse, columnID
} else {
viewName, id = ViewStandard, rowID
if !rowOK {
return nil, fmt.Errorf("Range() must specify %q", rowLabel)
}
// Parse start time.
@ -811,12 +748,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Union bitmaps across all time-based subframes.
row := &Row{}
for _, view := range ViewsByTimeRange(viewName, startTime, endTime, q) {
for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(index, frame, view, slice)
if f == nil {
continue
}
row = row.Union(f.Row(id))
row = row.Union(f.Row(rowID))
}
f.Stats.Count("range", 1, 1.0)
return row, nil
@ -1033,7 +970,6 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call,
// executeClearBit executes a ClearBit() call.
func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
view, _ := c.Args["view"].(string)
frame, ok := c.Args["frame"].(string)
if !ok {
return false, errors.New("ClearBit() frame required")
@ -1064,31 +1000,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel)
}
// Clear bits for each view.
switch view {
case ViewStandard:
return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt)
case ViewInverse:
return e.executeClearBitView(ctx, index, c, f, view, rowID, colID, opt)
case "":
var ret bool
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
if f.InverseEnabled() {
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewInverse, rowID, colID, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
}
return ret, nil
default:
return false, fmt.Errorf("invalid view: %s", view)
}
return e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt)
}
// executeClearBitView executes a ClearBit() call for a single view.
@ -1123,7 +1035,6 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql
// executeSetBit executes a SetBit() call.
func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
view, _ := c.Args["view"].(string)
frame, ok := c.Args["frame"].(string)
if !ok {
return false, errors.New("SetBit() field required: frame")
@ -1164,31 +1075,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
timestamp = &t
}
// Set bits for each view.
switch view {
case ViewStandard:
return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt)
case ViewInverse:
return e.executeSetBitView(ctx, index, c, f, view, rowID, colID, timestamp, opt)
case "":
var ret bool
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
if f.InverseEnabled() {
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
}
return ret, nil
default:
return false, fmt.Errorf("invalid view: %s", view)
}
return e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt)
}
// executeSetBitView executes a SetBit() call for a specific view.
@ -1319,7 +1206,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql.
if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil {
return err
}
frame.Stats.Count("SetBitmapAttrs", 1, 1.0)
frame.Stats.Count("SetRowAttrs", 1, 1.0)
// Do not forward call if this is already being forwarded.
if opt.Remote {
@ -1404,7 +1291,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal
if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil {
return nil, err
}
frame.Stats.Count("SetBitmapAttrs", 1, 1.0)
frame.Stats.Count("SetRowAttrs", 1, 1.0)
}
// Do not forward call if this is already being forwarded.
@ -1700,9 +1587,9 @@ type mapResponse struct {
// ExecOptions represents an execution context for a single Execute() call.
type ExecOptions struct {
Remote bool
ExcludeAttrs bool
ExcludeBits bool
Remote bool
ExcludeRowAttrs bool
ExcludeColumns bool
}
// decodeError returns an error representation of s if s is non-blank.

146
executor_test.go
View file

@ -33,7 +33,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
f, err := index.CreateFrame("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
@ -54,26 +54,26 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", bits)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit bits.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil {
// Inhibit column attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil {
// Inhibit row attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
@ -83,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if _, err := index.CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -100,14 +100,6 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
})
}
@ -124,8 +116,8 @@ func TestExecutor_Execute_Difference(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -156,8 +148,8 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -186,8 +178,8 @@ func TestExecutor_Execute_Union(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -200,8 +192,8 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -219,8 +211,8 @@ func TestExecutor_Execute_Xor(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -255,7 +247,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatal(err)
} else {
if !res[0].(bool) {
t.Fatalf("expected bit changed")
t.Fatalf("expected column changed")
}
}
@ -266,7 +258,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
t.Fatal(err)
} else {
if res[0].(bool) {
t.Fatalf("expected bit unchanged")
t.Fatalf("expected column unchanged")
}
}
}
@ -409,12 +401,12 @@ func TestExecutor_Execute_TopN(t *testing.T) {
defer hldr.Close()
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set bits for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
} else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
@ -431,7 +423,6 @@ func TestExecutor_Execute_TopN(t *testing.T) {
}
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewInverse, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
@ -445,24 +436,13 @@ func TestExecutor_Execute_TopN(t *testing.T) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("Inverse", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{ID: SliceWidth, Count: 3},
{ID: 0, Count: 2},
}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
}
func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two slices.
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2)
@ -520,7 +500,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
// Set columns for rows 0, 10, & 20 across two slices.
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth)
@ -609,7 +589,6 @@ func TestExecutor_Execute_MinMax(t *testing.T) {
}
if _, err := idx.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
Fields: []*pilosa.Field{
{Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100},
},
@ -759,13 +738,12 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
InverseEnabled: true,
TimeQuantum: pilosa.TimeQuantum("YMDH"),
TimeQuantum: pilosa.TimeQuantum("YMDH"),
}); err != nil {
t.Fatal(err)
}
// Set bits.
// Set columns.
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00")
SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00")
@ -784,19 +762,11 @@ func TestExecutor_Execute_Range(t *testing.T) {
t.Run("Standard", func(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
t.Run("Inverse", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) {
t.Fatalf("unexpected bits: %+v", bits)
}
})
}
// Ensure a Range(field) query can be executed.
@ -854,7 +824,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("EQ", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -863,28 +833,28 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
// NEQ null
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ <int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
// NEQ -<int>
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
//t.Fatalf("unexpected result: %s", spew.Sdump(result))
t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Bits())
t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Columns())
}
})
t.Run("LT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -892,7 +862,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("LTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -900,7 +870,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("GT", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -908,7 +878,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("GTE", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -916,7 +886,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("BETWEEN", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -925,7 +895,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("FieldNotNull", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -933,7 +903,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("BelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -941,7 +911,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("AboveMax", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) {
} else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
@ -949,16 +919,16 @@ func TestExecutor_Execute_FieldRange(t *testing.T) {
t.Run("LTAboveMax", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo < 200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits()))
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns()))
}
})
t.Run("GTBelowMin", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo > -200)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits()))
} else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) {
t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns()))
}
})
@ -999,7 +969,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
t.Fatalf("unexpected slices: %+v", slices)
}
// Set bits in slice 0 & 2.
// Set columns in slice 0 & 2.
r := pilosa.NewRow(
(0*SliceWidth)+1,
(0*SliceWidth)+2,
@ -1018,8 +988,8 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
}
}
@ -1058,7 +1028,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
}
}
// Ensure a remote query can set bits on multiple nodes.
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
c := test.NewCluster(2)
c.ReplicaN = 2
@ -1101,7 +1071,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
t.Fatal(err)
}
// Verify that one bit is set on both node's holder.
// Verify that one column is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
@ -1110,7 +1080,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
}
// Ensure a remote query can set bits on multiple nodes.
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
c := test.NewCluster(2)
c.ReplicaN = 2
@ -1155,7 +1125,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
t.Fatal(err)
}
// Verify that one bit is set on both node's holder.
// Verify that one column is set on both node's holder.
if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 {
t.Fatalf("unexpected local count: %d", n)
}
@ -1247,7 +1217,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
index.CreateFrame("f", pilosa.FrameOptions{})
targetAttrs := map[string]interface{}{
"foo": "bar",
}

14
fragment.go
View file

@ -393,7 +393,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) {
// Determine the position of the bit in the storage.
pos, err := f.pos(rowID, columnID)
if err != nil {
return false, errors.Wrap(err, "getting bit ops")
return false, errors.Wrap(err, "getting bit pos")
}
// Write to storage.
@ -585,7 +585,7 @@ func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uin
// FieldSum returns the sum of a given field as well as the number of columns involved.
// A bitmap can be passed in to optionally filter the computed columns.
func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) {
// Compute count based on the existence bit.
// Compute count based on the existence row.
row := f.Row(uint64(bitDepth))
if filter != nil {
count = row.IntersectionCount(filter)
@ -629,7 +629,7 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
ii := i - 1 // allow for uint range: (bitDepth-1) to 0
row := f.Row(uint64(ii))
x := consider.Difference(row)
@ -662,7 +662,7 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err
}
for i := bitDepth; i > uint(0); i-- {
ii := i - 1 // allow for uint range: (bitdepth-1) to 0
ii := i - 1 // allow for uint range: (bitDepth-1) to 0
row := f.Row(uint64(ii))
x := row.Intersect(consider)
@ -842,7 +842,7 @@ func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax u
}
// LTE predicateMin
// If bit is zero then remove all set columns not in excluded bitmap.
// If bit is zero then remove all set bits not in excluded bitmap.
if bit2 == 0 {
b = b.Difference(row.Difference(keep2))
} else {
@ -996,13 +996,13 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) {
// If it's too low then don't try finding anymore pairs.
threshold := results.Pairs[0].Count
// If the row doesn't have enough bits set before the intersection
// If the row doesn't have enough columns set before the intersection
// then we can assume that any remaining rows also have a count too low.
if threshold < opt.MinThreshold || cnt < threshold {
break
}
// Calculate the intersecting bit count and skip if it's below our
// Calculate the intersecting column count and skip if it's below our
// last row in our current result set.
count := opt.Src.IntersectionCount(f.Row(rowID))
if count < threshold {

92
fragment_test.go
View file

@ -176,10 +176,10 @@ func TestFragment_SetFieldValue(t *testing.T) {
})
t.Run("QuickCheck", func(t *testing.T) {
if err := quick.Check(func(bitDepth uint, columnN uint64, values []uint64) bool {
if err := quick.Check(func(bitDepth uint, bitN uint64, values []uint64) bool {
// Limit bit depth & maximum values.
bitDepth = (bitDepth % 62) + 1
columnN = (columnN % 99) + 1
bitN = (bitN % 99) + 1
for i := range values {
values[i] = values[i] % (1 << bitDepth)
@ -191,7 +191,7 @@ func TestFragment_SetFieldValue(t *testing.T) {
// Set values.
m := make(map[uint64]int64)
for _, value := range values {
columnID := value % columnN
columnID := value % bitN
m[columnID] = int64(value)
@ -206,9 +206,9 @@ func TestFragment_SetFieldValue(t *testing.T) {
if err != nil {
t.Fatal(err)
} else if value != int64(v) {
t.Fatalf("value mismatch: column=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v)
t.Fatalf("value mismatch: columnID=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v)
} else if !exists {
t.Fatalf("value should exist: column=%d", columnID)
t.Fatalf("value should exist: columnID=%d", columnID)
}
}
@ -353,8 +353,8 @@ func TestFragment_FieldRange(t *testing.T) {
// Query for equality.
if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -376,8 +376,8 @@ func TestFragment_FieldRange(t *testing.T) {
// Query for inequality.
if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -400,32 +400,32 @@ func TestFragment_FieldRange(t *testing.T) {
t.Fatal(err)
}
// Query for fields less than (ending with set bit).
// Query for fields less than (ending with set column).
if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields less than (ending with unset bit).
// Query for fields less than (ending with unset column).
if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields less than or equal to (ending with set bit).
// Query for fields less than or equal to (ending with set column).
if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields less than or equal to (ending with unset bit).
// Query for fields less than or equal to (ending with unset column).
if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -451,29 +451,29 @@ func TestFragment_FieldRange(t *testing.T) {
// Query for fields greater than (ending with unset bit).
if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than (ending with set bit).
if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with unset bit).
if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with set bit).
if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
@ -496,32 +496,32 @@ func TestFragment_FieldRange(t *testing.T) {
t.Fatal(err)
}
// Query for fields greater than (ending with unset bit).
// Query for fields greater than (ending with unset column).
if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than (ending with set bit).
// Query for fields greater than (ending with set column).
if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with unset bit).
// Query for fields greater than or equal to (ending with unset column).
if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
// Query for fields greater than or equal to (ending with set bit).
// Query for fields greater than or equal to (ending with set column).
if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 4000}) {
t.Fatalf("unexpected bits: %+v", b.Bits())
} else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) {
t.Fatalf("unexpected columns: %+v", b.Columns())
}
})
}
@ -663,7 +663,7 @@ func TestFragment_TopN_Intersect(t *testing.T) {
}
}
// Ensure a fragment can return top rows that have many bits set.
// Ensure a fragment can return top rows that have many columns set.
func TestFragment_TopN_Intersect_Large(t *testing.T) {
if testing.Short() {
t.Skip("short mode")
@ -1011,8 +1011,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
}
// Verify data in other fragment.
if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected bits: %+v", a)
if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected columns: %+v", a)
}
// Close and reopen the fragment & verify the data.
@ -1020,8 +1020,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
t.Fatal(err)
} else if n := f1.Cache().Len(); n != 1 {
t.Fatalf("unexpected cache size (reopen): %d", n)
} else if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected bits (reopen): %+v", a)
} else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) {
t.Fatalf("unexpected columns (reopen): %+v", a)
}
}

110
frame.go
View file

@ -31,8 +31,7 @@ import (
// Default frame settings.
const (
DefaultCacheType = CacheTypeRanked
DefaultInverseEnabled = false
DefaultCacheType = CacheTypeRanked
// Default ranked frame cache
DefaultCacheSize = 50000
@ -54,11 +53,10 @@ type Frame struct {
Stats StatsClient
// Frame options.
inverseEnabled bool
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
fields []*Field
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
fields []*Field
Logger Logger
}
@ -82,9 +80,8 @@ func NewFrame(path, index, name string) (*Frame, error) {
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
inverseEnabled: DefaultInverseEnabled,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
//timeQuantum
//fields
@ -111,37 +108,18 @@ func (f *Frame) MaxSlice() uint64 {
var max uint64
for _, view := range f.views {
if view.name == ViewInverse {
continue
} else if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max {
if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max {
max = viewMaxSlice
}
}
return max
}
// MaxInverseSlice returns the max inverse slice in the frame.
func (f *Frame) MaxInverseSlice() uint64 {
f.mu.RLock()
defer f.mu.RUnlock()
view := f.views[ViewInverse]
if view == nil {
return 0
}
return view.MaxSlice()
}
// CacheType returns the caching mode for the frame.
func (f *Frame) CacheType() string {
return f.cacheType
}
// InverseEnabled returns true if an inverse view is available.
func (f *Frame) InverseEnabled() bool {
return f.inverseEnabled
}
// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update.
// defaults to DefaultCacheSize 50000
func (f *Frame) SetCacheSize(v uint32) error {
@ -179,11 +157,10 @@ func (f *Frame) Options() FrameOptions {
func (f *Frame) options() FrameOptions {
return FrameOptions{
InverseEnabled: f.inverseEnabled,
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
Fields: f.fields,
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
Fields: f.fields,
}
}
@ -255,7 +232,6 @@ func (f *Frame) loadMeta() error {
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta"))
if os.IsNotExist(err) {
f.inverseEnabled = DefaultInverseEnabled
f.cacheType = DefaultCacheType
f.cacheSize = DefaultCacheSize
f.timeQuantum = ""
@ -270,7 +246,6 @@ func (f *Frame) loadMeta() error {
}
// Copy metadata fields.
f.inverseEnabled = pb.InverseEnabled
f.cacheType = pb.CacheType
if f.cacheType == "" {
f.cacheType = DefaultCacheType
@ -532,11 +507,6 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
// createViewIfNotExistsBase returns the named view, creating it if necessary.
// The returned bool indicates whether the view was created or not.
func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) {
// Don't create inverse views if they are not enabled.
if !f.InverseEnabled() && IsInverseView(name) {
return nil, false, ErrFrameInverseDisabled
}
f.mu.Lock()
defer f.mu.Unlock()
@ -840,16 +810,14 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
timestamp = timestamps[i]
}
var standard, inverse []string
var standard []string
if timestamp == nil {
standard = []string{ViewStandard}
inverse = []string{ViewInverse}
} else {
standard = ViewsByTime(ViewStandard, *timestamp, q)
// In order to match the logic of `SetBit()`, we want bits
// with timestamps to write to both time and standard views.
standard = append(standard, ViewStandard)
inverse = ViewsByTime(ViewInverse, *timestamp, q)
}
// Attach bit to each standard view.
@ -860,34 +828,10 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
data.ColumnIDs = append(data.ColumnIDs, columnID)
dataByFragment[key] = data
}
if f.inverseEnabled {
// Attach reversed bits to each inverse view.
for _, name := range inverse {
key := importKey{View: name, Slice: rowID / SliceWidth}
data := dataByFragment[key]
data.RowIDs = append(data.RowIDs, columnID) // reversed
data.ColumnIDs = append(data.ColumnIDs, rowID) // reversed
dataByFragment[key] = data
}
}
}
// Import into each fragment.
for key, data := range dataByFragment {
// Skip inverse data if inverse is not enabled.
if !f.inverseEnabled && IsInverseView(key.View) {
continue
}
// Re-sort data for inverse views.
if IsInverseView(key.View) {
sort.Sort(importBitSet{
rowIDs: data.RowIDs,
columnIDs: data.ColumnIDs,
})
}
view, err := f.CreateViewIfNotExists(key.View)
if err != nil {
return errors.Wrap(err, "creating view")
@ -1003,12 +947,10 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FrameOptions represents options to set when initializing a frame.
type FrameOptions struct {
InverseEnabled bool `json:"inverseEnabled,omitempty"`
RangeEnabled bool `json:"rangeEnabled,omitempty"` // deprecated, will be removed
CacheType string `json:"cacheType,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Fields []*Field `json:"fields,omitempty"`
CacheType string `json:"cacheType,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Fields []*Field `json:"fields,omitempty"`
}
// Encode converts o into its internal representation.
@ -1021,11 +963,10 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta {
return nil
}
return &internal.FrameMeta{
InverseEnabled: o.InverseEnabled,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
TimeQuantum: string(o.TimeQuantum),
Fields: encodeFields(o.Fields),
CacheType: o.CacheType,
CacheSize: o.CacheSize,
TimeQuantum: string(o.TimeQuantum),
Fields: encodeFields(o.Fields),
}
}
@ -1034,11 +975,10 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions {
return nil
}
return &FrameOptions{
InverseEnabled: options.InverseEnabled,
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Fields: decodeFields(options.Fields),
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Fields: decodeFields(options.Fields),
}
}

111
handler.go
View file

@ -81,8 +81,8 @@ func NewHandler() *Handler {
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")
h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse")
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits")
h.validators["GetSliceMax"] = queryValidationSpecRequired()
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice")
@ -119,7 +119,6 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups)
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
@ -134,7 +133,6 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST").Name("PostFragmentData")
router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/import", handler.handlePostImport).Methods("POST")
router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST")
@ -147,7 +145,6 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE")
router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore")
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET")
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE")
@ -303,7 +300,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{
Standard: h.API.MaxSlices(r.Context()),
Inverse: h.API.MaxInverseSlices(r.Context()),
}); err != nil {
h.Logger.Printf("write slices-max response error: %s", err)
}
@ -311,7 +307,6 @@ func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
type getSlicesMaxResponse struct {
Standard map[string]uint64 `json:"standard"`
Inverse map[string]uint64 `json:"inverse"`
}
// handleGetIndexes handles GET /index request.
@ -821,11 +816,11 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) {
}
return &QueryRequest{
Query: query,
Slices: slices,
ColumnAttrs: q.Get("columnAttrs") == "true",
ExcludeAttrs: q.Get("excludeAttrs") == "true",
ExcludeBits: q.Get("excludeBits") == "true",
Query: query,
Slices: slices,
ColumnAttrs: q.Get("columnAttrs") == "true",
ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true",
ExcludeColumns: q.Get("excludeColumns") == "true",
}, nil
}
@ -1017,48 +1012,6 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
}
}
// handleGetFragmentData handles GET /fragment/data requests.
func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) {
// Read slice parameter.
q := r.URL.Query()
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
if err != nil {
http.Error(w, "slice required", http.StatusBadRequest)
return
}
// Retrieve fragment from holder.
f, err := h.API.MarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
// Stream fragment to response body.
if _, err := f.WriteTo(w); err != nil {
h.Logger.Printf("fragment backup error: %s", err)
}
}
// handlePostFragmentData handles POST /fragment/data requests.
func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) {
// Read slice parameter.
q := r.URL.Query()
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
if err != nil {
http.Error(w, "slice required", http.StatusBadRequest)
return
}
if err = h.API.UnmarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil {
if errors.Cause(err) == ErrFrameNotFound {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
}
// handleGetFragmentBlockData handles GET /fragment/block/data requests.
func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) {
buf, err := h.API.FragmentBlockData(r.Context(), r.Body)
@ -1111,38 +1064,6 @@ type getFragmentBlocksResponse struct {
Blocks []FragmentBlock `json:"blocks"`
}
// handlePostFrameRestore handles POST /frame/restore requests.
func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
q := r.URL.Query()
hostStr := q.Get("host")
// Validate query parameters.
if hostStr == "" {
http.Error(w, "host required", http.StatusBadRequest)
return
}
host, err := NewURIFromAddress(hostStr)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
}
err = h.API.RestoreFrame(r.Context(), indexName, frameName, host)
switch errors.Cause(err) {
case nil:
break
case ErrFrameNotFound:
fallthrough
case ErrFragmentNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
// handleGetVersion handles /version requests.
func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) {
err := json.NewEncoder(w).Encode(struct {
@ -1181,10 +1102,10 @@ type QueryRequest struct {
ColumnAttrs bool
// Do not return row attributes, if true.
ExcludeAttrs bool
ExcludeRowAttrs bool
// Do not return bits, if true.
ExcludeBits bool
// Do not return columns, if true.
ExcludeColumns bool
// If true, indicates that query is part of a larger distributed query.
// If false, this request is on the originating node.
@ -1193,12 +1114,12 @@ type QueryRequest struct {
func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest {
req := &QueryRequest{
Query: pb.Query,
Slices: pb.Slices,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeAttrs: pb.ExcludeAttrs,
ExcludeBits: pb.ExcludeBits,
Query: pb.Query,
Slices: pb.Slices,
ColumnAttrs: pb.ColumnAttrs,
Remote: pb.Remote,
ExcludeRowAttrs: pb.ExcludeRowAttrs,
ExcludeColumns: pb.ExcludeColumns,
}
return req

4
handler_internal_test.go
View file

@ -66,8 +66,8 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) {
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
{json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}},
{json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}},
{json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"},
{json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}},
{json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
}
for _, test := range tests {

116
handler_test.go
View file

@ -84,12 +84,10 @@ func TestHandler_Schema(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
@ -107,8 +105,8 @@ func TestHandler_Schema(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -123,12 +121,10 @@ func TestHandler_Status(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
@ -209,48 +205,7 @@ func TestHandler_MaxSlices(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can return the maxslice map for the inverse views.
func TestHandler_MaxSlices_Inverse(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true})
if err != nil {
t.Fatal(err)
}
if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+1, nil); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+2, nil); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (3*SliceWidth)+4, nil); err != nil {
t.Fatal(err)
}
f1, err := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true})
if err != nil {
t.Fatal(err)
}
if _, err := f1.SetBit(pilosa.ViewStandard, 40, (0*SliceWidth)+1, nil); err != nil {
t.Fatal(err)
} else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+2, nil); err != nil {
t.Fatal(err)
} else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+4, nil); err != nil {
t.Fatal(err)
}
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -419,7 +374,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -452,7 +407,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -484,8 +439,8 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
@ -541,8 +496,8 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) {
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
@ -1101,55 +1056,6 @@ type FrameFields struct {
Fields []pilosa.Field
}
// Ensure the handler can backup a fragment and then restore it.
func TestHandler_Fragment_BackupRestore(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set bits in the index.
f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
f0.MustSetBits(100, 1, 2, 3)
// Begin backing up from slice i/f/0.
resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Ensure response came back OK.
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected backup status code: %d", resp.StatusCode)
}
// Create frame.
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Restore backup to slice x/y/0.
if resp, err := http.Post(s.URL+"/fragment/data?index=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
resp.Body.Close()
t.Fatalf("unexpected restore status code: %d", resp.StatusCode)
} else {
resp.Body.Close()
}
// Verify data is correctly restored.
f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0)
if f1 == nil {
t.Fatal("fragment x/y/standard/0 not created")
} else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) {
t.Fatalf("unexpected restored bits: %+v", bits)
}
}
// Ensure the handler can retrieve the version.
func TestHandler_Version(t *testing.T) {
hldr := test.MustOpenHolder()

10
holder.go
View file

@ -209,15 +209,6 @@ func (h *Holder) MaxSlices() map[string]uint64 {
return a
}
// MaxInverseSlices returns MaxInverseSlice map for all indexes.
func (h *Holder) MaxInverseSlices() map[string]uint64 {
a := make(map[string]uint64)
for _, index := range h.Indexes() {
a[index.Name()] = index.MaxInverseSlice()
}
return a
}
// Schema returns schema information for all indexes, frames, and views.
func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
@ -270,7 +261,6 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
func (h *Holder) EncodeMaxSlices() *internal.MaxSlices {
return &internal.MaxSlices{
Standard: h.MaxSlices(),
Inverse: h.MaxInverseSlices(),
}
}

88
holder_test.go
View file

@ -456,29 +456,29 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0, hldr1} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
}
@ -554,29 +554,29 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
a := f.Row(9).Bits()
a := f.Row(9).Columns()
if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a)
t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a)
}
if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a)
if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) {
t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a)
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
@ -597,16 +597,16 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
// Verify data is the same on both nodes.
for i, hldr := range []*test.Holder{hldr0} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
} else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected bits(%d/2): %+v", i, a)
} else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/3): %+v", i, a)
} else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected bits(%d/120): %+v", i, a)
} else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected bits(%d/200): %+v", i, a)
if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected columns(%d/0): %+v", i, a)
} else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) {
t.Fatalf("unexpected columns(%d/2): %+v", i, a)
} else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/3): %+v", i, a)
} else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) {
t.Fatalf("unexpected columns(%d/120): %+v", i, a)
} else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) {
t.Fatalf("unexpected columns(%d/200): %+v", i, a)
}
f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1)
@ -615,8 +615,8 @@ func TestHolderCleaner_CleanHolder(t *testing.T) {
}
f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2)
if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected bits(%d/y/z): %+v", i, a)
if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) {
t.Fatalf("unexpected columns(%d/y/z): %+v", i, a)
}
}
}

37
index.go
View file

@ -38,8 +38,7 @@ type Index struct {
frames map[string]*Frame
// Max Slice on any node in the cluster, according to this node.
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
remoteMaxSlice uint64
NewAttrStore func(string) AttrStore
@ -64,8 +63,7 @@ func NewIndex(path, name string) (*Index, error) {
name: name,
frames: make(map[string]*Frame),
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
remoteMaxSlice: 0,
NewAttrStore: NewNopAttrStore,
columnAttrStore: NopAttrStore,
@ -236,30 +234,6 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) {
i.remoteMaxSlice = newmax
}
// MaxInverseSlice returns the max inverse slice in the index according to this node.
func (i *Index) MaxInverseSlice() uint64 {
if i == nil {
return 0
}
i.mu.RLock()
defer i.mu.RUnlock()
max := i.remoteMaxInverseSlice
for _, f := range i.frames {
if slice := f.MaxInverseSlice(); slice > max {
max = slice
}
}
return max
}
// SetRemoteMaxInverseSlice sets the remote max inverse slice value received from another node.
func (i *Index) SetRemoteMaxInverseSlice(v uint64) {
i.mu.Lock()
defer i.mu.Unlock()
i.remoteMaxInverseSlice = v
}
// FramePath returns the path to a frame in the index.
func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
@ -325,11 +299,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
return nil, ErrInvalidCacheType
}
// Validate mutually exclusive options if ranges are enabled.
if opt.RangeEnabled {
i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame")
}
// Validate fields.
for _, field := range opt.Fields {
if err := ValidateField(field); err != nil {
@ -364,8 +333,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
f.cacheSize = opt.CacheSize
}
f.inverseEnabled = opt.InverseEnabled
// Set fields.
f.fields = opt.Fields

49
index_test.go
View file

@ -67,14 +67,13 @@ func TestIndex_CreateFrame(t *testing.T) {
})
// Ensure frame can include range columns.
t.Run("RangeEnabled", func(t *testing.T) {
t.Run("BSIFields", func(t *testing.T) {
t.Run("OK", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
// Create frame with schema and verify it exists.
if f, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: false,
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20},
{Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21},
@ -99,49 +98,6 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
t.Run("ErrInverseRangeAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
frame, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true,
InverseEnabled: true,
Fields: []*pilosa.Field{
&pilosa.Field{
Name: "myfield",
Type: pilosa.FieldTypeInt,
Min: -20,
Max: 100,
},
},
})
if err != nil {
t.Fatal(err)
}
ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetFieldValue(1, "myfield", 87)
if !ch || err != nil {
t.Fatal(ch, err)
}
views := frame.Views()
if len(views) != 3 {
var names string
for _, v := range views {
names = names + v.Name() + " "
}
t.Fatalf("Unexpected views: %s", names)
}
})
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
@ -153,7 +109,7 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
t.Run("RangeEnabledWithCacheTypeNone", func(t *testing.T) {
t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
@ -208,7 +164,6 @@ func TestIndex_CreateFrame(t *testing.T) {
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation
Fields: []*pilosa.Field{
{Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50},
},

411
internal/private.pb.go
View file

@ -70,12 +70,10 @@ func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} }
type FrameMeta struct {
InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"`
CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"`
TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
RangeEnabled bool `protobuf:"varint,6,opt,name=RangeEnabled,proto3" json:"RangeEnabled,omitempty"`
Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"`
CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"`
TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"`
}
func (m *FrameMeta) Reset() { *m = FrameMeta{} }
@ -83,13 +81,6 @@ func (m *FrameMeta) String() string { return proto.CompactTextString(
func (*FrameMeta) ProtoMessage() {}
func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} }
func (m *FrameMeta) GetInverseEnabled() bool {
if m != nil {
return m.InverseEnabled
}
return false
}
func (m *FrameMeta) GetCacheType() string {
if m != nil {
return m.CacheType
@ -111,13 +102,6 @@ func (m *FrameMeta) GetTimeQuantum() string {
return ""
}
func (m *FrameMeta) GetRangeEnabled() bool {
if m != nil {
return m.RangeEnabled
}
return false
}
func (m *FrameMeta) GetFields() []*Field {
if m != nil {
return m.Fields
@ -231,7 +215,6 @@ func (m *Cache) GetIDs() []uint64 {
type MaxSlices struct {
Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
Inverse map[string]uint64 `protobuf:"bytes,2,rep,name=Inverse" json:"Inverse,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
}
func (m *MaxSlices) Reset() { *m = MaxSlices{} }
@ -246,17 +229,9 @@ func (m *MaxSlices) GetStandard() map[string]uint64 {
return nil
}
func (m *MaxSlices) GetInverse() map[string]uint64 {
if m != nil {
return m.Inverse
}
return nil
}
type CreateSliceMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
IsInverse bool `protobuf:"varint,3,opt,name=IsInverse,proto3" json:"IsInverse,omitempty"`
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
}
func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} }
@ -278,13 +253,6 @@ func (m *CreateSliceMessage) GetSlice() uint64 {
return 0
}
func (m *CreateSliceMessage) GetIsInverse() bool {
if m != nil {
return m.IsInverse
}
return false
}
type DeleteIndexMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
}
@ -1067,16 +1035,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if m.InverseEnabled {
dAtA[i] = 0x10
i++
if m.InverseEnabled {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if len(m.CacheType) > 0 {
dAtA[i] = 0x1a
i++
@ -1094,16 +1052,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum)))
i += copy(dAtA[i:], m.TimeQuantum)
}
if m.RangeEnabled {
dAtA[i] = 0x30
i++
if m.RangeEnabled {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if len(m.Fields) > 0 {
for _, msg := range m.Fields {
dAtA[i] = 0x3a
@ -1307,22 +1255,6 @@ func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPrivate(dAtA, i, uint64(v))
}
}
if len(m.Inverse) > 0 {
for k, _ := range m.Inverse {
dAtA[i] = 0x12
i++
v := m.Inverse[k]
mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v))
i = encodeVarintPrivate(dAtA, i, uint64(mapSize))
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(k)))
i += copy(dAtA[i:], k)
dAtA[i] = 0x10
i++
i = encodeVarintPrivate(dAtA, i, uint64(v))
}
}
return i, nil
}
@ -1352,16 +1284,6 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Slice))
}
if m.IsInverse {
dAtA[i] = 0x18
i++
if m.IsInverse {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -2324,9 +2246,6 @@ func (m *IndexMeta) Size() (n int) {
func (m *FrameMeta) Size() (n int) {
var l int
_ = l
if m.InverseEnabled {
n += 2
}
l = len(m.CacheType)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
@ -2338,9 +2257,6 @@ func (m *FrameMeta) Size() (n int) {
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
}
if m.RangeEnabled {
n += 2
}
if len(m.Fields) > 0 {
for _, e := range m.Fields {
l = e.Size()
@ -2428,14 +2344,6 @@ func (m *MaxSlices) Size() (n int) {
n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize))
}
}
if len(m.Inverse) > 0 {
for k, v := range m.Inverse {
_ = k
_ = v
mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v))
n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize))
}
}
return n
}
@ -2449,9 +2357,6 @@ func (m *CreateSliceMessage) Size() (n int) {
if m.Slice != 0 {
n += 1 + sovPrivate(uint64(m.Slice))
}
if m.IsInverse {
n += 2
}
return n
}
@ -2957,26 +2862,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error {
return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.InverseEnabled = bool(v != 0)
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field CacheType", wireType)
@ -3054,26 +2939,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error {
}
m.TimeQuantum = string(dAtA[iNdEx:postIndex])
iNdEx = postIndex
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field RangeEnabled", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.RangeEnabled = bool(v != 0)
case 7:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType)
@ -3802,113 +3667,6 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error {
}
m.Standard[mapkey] = mapvalue
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Inverse", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Inverse == nil {
m.Inverse = make(map[string]uint64)
}
var mapkey string
var mapvalue uint64
for iNdEx < postIndex {
entryPreIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
if fieldNum == 1 {
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthPrivate
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
return io.ErrUnexpectedEOF
}
mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
} else if fieldNum == 2 {
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
mapvalue |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
} else {
iNdEx = entryPreIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > postIndex {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
m.Inverse[mapkey] = mapvalue
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -4007,26 +3765,6 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IsInverse", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.IsInverse = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7259,75 +6997,70 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 1112 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6f, 0x1b, 0x45,
0x14, 0x67, 0xbd, 0x6b, 0x27, 0x7e, 0xae, 0x53, 0x67, 0x5a, 0xca, 0xb6, 0xaa, 0x82, 0x19, 0x15,
0x6a, 0x38, 0x44, 0x25, 0xbd, 0x40, 0xa1, 0x52, 0x95, 0x38, 0x15, 0x8b, 0x48, 0x04, 0xe3, 0xa4,
0x07, 0x24, 0x90, 0x26, 0xf6, 0x28, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0xf9, 0xe8, 0x81, 0x33,
0x17, 0xee, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x01, 0x42, 0xe1, 0x0f, 0x01, 0xbd, 0x37, 0xb3, 0x1f,
0xb1, 0x9d, 0xa6, 0x04, 0x6e, 0xf3, 0x3e, 0xe7, 0xf7, 0x3e, 0x67, 0x17, 0xda, 0x93, 0x34, 0x3c,
0x96, 0x5a, 0xad, 0x4f, 0xd2, 0x44, 0x27, 0x6c, 0x39, 0x8c, 0xb5, 0x4a, 0x63, 0x19, 0xf1, 0x16,
0x34, 0x83, 0x78, 0xa4, 0x4e, 0x77, 0x94, 0x96, 0xfc, 0x0f, 0x07, 0x9a, 0xcf, 0x53, 0x39, 0x56,
0x48, 0xb1, 0x0f, 0x60, 0x25, 0x88, 0x8f, 0x55, 0x9a, 0xa9, 0xed, 0x58, 0x1e, 0x44, 0x6a, 0xe4,
0xd7, 0xba, 0x4e, 0x6f, 0x59, 0xcc, 0x70, 0xd9, 0x7d, 0x68, 0x6e, 0xc9, 0xe1, 0x4b, 0xb5, 0x77,
0x36, 0x51, 0xbe, 0xdb, 0x75, 0x7a, 0x4d, 0x51, 0x32, 0x0a, 0xe9, 0x20, 0x7c, 0xa5, 0x7c, 0xaf,
0xeb, 0xf4, 0xda, 0xa2, 0x64, 0xb0, 0x2e, 0xb4, 0xf6, 0xc2, 0xb1, 0xfa, 0x66, 0x2a, 0x63, 0x3d,
0x1d, 0xfb, 0x75, 0xb2, 0xae, 0xb2, 0x18, 0x87, 0x1b, 0x42, 0xc6, 0x87, 0x05, 0x86, 0x06, 0x61,
0xb8, 0xc0, 0x63, 0x0f, 0xa1, 0xf1, 0x3c, 0x54, 0xd1, 0x28, 0xf3, 0x97, 0xba, 0x6e, 0xaf, 0xb5,
0x71, 0x73, 0x3d, 0x8f, 0x6f, 0x9d, 0xf8, 0xc2, 0x8a, 0x39, 0x87, 0x95, 0x60, 0x3c, 0x49, 0x52,
0x2d, 0x54, 0x36, 0x49, 0xe2, 0x4c, 0xb1, 0x0e, 0xb8, 0xdb, 0x69, 0xea, 0x3b, 0x74, 0x31, 0x1e,
0xf9, 0x8f, 0xd0, 0xd9, 0x8c, 0x92, 0xe1, 0x51, 0x5f, 0x6a, 0x29, 0xd4, 0x0f, 0x53, 0x95, 0x69,
0x76, 0x1b, 0xea, 0x94, 0x25, 0xab, 0x67, 0x08, 0xe4, 0x52, 0xb6, 0x28, 0x2f, 0x4d, 0x61, 0x08,
0xe4, 0x92, 0x3d, 0xa5, 0xc2, 0x13, 0x86, 0x40, 0xee, 0x20, 0x0a, 0x87, 0x26, 0x05, 0x9e, 0x30,
0x04, 0x63, 0xe0, 0xbd, 0x08, 0xd5, 0x89, 0x8d, 0x9b, 0xce, 0x3c, 0x80, 0xd5, 0xca, 0xfd, 0x16,
0xe6, 0x1d, 0x68, 0x88, 0xe4, 0x24, 0xe8, 0x67, 0xbe, 0xd3, 0x75, 0x7b, 0x9e, 0xb0, 0x14, 0x65,
0x37, 0x89, 0xa6, 0xe3, 0x18, 0x45, 0x35, 0x12, 0x95, 0x0c, 0x7e, 0x17, 0xea, 0x94, 0x6a, 0x8c,
0xb2, 0xb4, 0xc5, 0x23, 0xff, 0xdb, 0x81, 0xe6, 0x8e, 0x3c, 0x25, 0x18, 0x19, 0x7b, 0x0a, 0xcb,
0x03, 0x2d, 0xe3, 0x91, 0x4c, 0x47, 0xa4, 0xd4, 0xda, 0x78, 0xaf, 0x4c, 0x61, 0xa1, 0xb6, 0x9e,
0xeb, 0x6c, 0xc7, 0x3a, 0x3d, 0x13, 0x85, 0x09, 0x7b, 0x02, 0x4b, 0xb6, 0x27, 0x08, 0x43, 0x6b,
0xa3, 0xbb, 0xc8, 0xba, 0x68, 0x1b, 0x34, 0xce, 0x0d, 0xee, 0x7d, 0x06, 0xed, 0x0b, 0x6e, 0x11,
0xeb, 0x91, 0x3a, 0xcb, 0x2b, 0x72, 0xa4, 0xce, 0x30, 0x77, 0xc7, 0x32, 0x9a, 0x9a, 0x3c, 0x7b,
0xc2, 0x10, 0x4f, 0x6a, 0x9f, 0x38, 0xf7, 0x9e, 0xc0, 0x8d, 0xaa, 0xd7, 0x7f, 0x63, 0xcb, 0xbf,
0x07, 0xb6, 0x95, 0x2a, 0xa9, 0x15, 0xc1, 0xdb, 0x51, 0x59, 0x26, 0x0f, 0xd5, 0xe5, 0x95, 0x36,
0xd5, 0xab, 0x55, 0xab, 0x77, 0x1f, 0x9a, 0x41, 0x96, 0x07, 0xee, 0x52, 0x5f, 0x96, 0x0c, 0xfe,
0x11, 0xb0, 0xbe, 0x8a, 0x94, 0x56, 0x76, 0xbe, 0x5e, 0xe3, 0x9f, 0x0f, 0x72, 0x2c, 0x57, 0xeb,
0xb2, 0x87, 0xe0, 0xe1, 0x78, 0x12, 0x94, 0xd6, 0xc6, 0xad, 0x32, 0xd3, 0xc5, 0x1c, 0x0b, 0x52,
0xe0, 0x61, 0xee, 0xd4, 0x8e, 0xf4, 0x15, 0x01, 0x2e, 0x68, 0xe5, 0xfc, 0x2a, 0x77, 0xf6, 0xaa,
0x62, 0x49, 0xd8, 0xab, 0x9e, 0xe5, 0xb1, 0x5e, 0xf7, 0x2a, 0x7e, 0x58, 0x80, 0xc5, 0x49, 0xbd,
0x0e, 0xd8, 0xf7, 0xa1, 0x4e, 0xb6, 0x16, 0xed, 0xdc, 0x0e, 0x30, 0x52, 0xfe, 0xa2, 0x80, 0x7a,
0xdd, 0x8b, 0x6e, 0x57, 0x2f, 0x6a, 0xe6, 0x7e, 0xbf, 0xb5, 0xba, 0x38, 0xd3, 0xbb, 0x68, 0x63,
0x3c, 0xd1, 0xf9, 0xf2, 0x9a, 0xcd, 0x24, 0x12, 0x7d, 0xe3, 0x12, 0xc8, 0x7c, 0xb7, 0xeb, 0xa2,
0x6f, 0x22, 0xf8, 0x63, 0x68, 0x0c, 0x86, 0x2f, 0xd5, 0x58, 0xb2, 0x0f, 0x71, 0xd2, 0x46, 0xea,
0x54, 0x65, 0x76, 0x4e, 0x6f, 0xce, 0xd4, 0x5f, 0xe4, 0x72, 0xde, 0xb7, 0x21, 0x5d, 0x02, 0xa8,
0x41, 0x57, 0x67, 0xbe, 0x37, 0xb7, 0x31, 0x91, 0x2f, 0xac, 0x98, 0x6f, 0x83, 0xbb, 0x2f, 0x02,
0xdc, 0x3f, 0x84, 0x20, 0xf7, 0x62, 0x29, 0xf4, 0xfd, 0x45, 0x92, 0x69, 0x9b, 0x20, 0x3a, 0x23,
0xef, 0xeb, 0x24, 0xd5, 0x94, 0x9e, 0xb6, 0xa0, 0x33, 0xff, 0x0e, 0xbc, 0xdd, 0x64, 0xa4, 0xd8,
0x0a, 0xd4, 0x82, 0xbe, 0xf5, 0x51, 0x0b, 0xfa, 0xec, 0x5d, 0x72, 0x6f, 0xf3, 0xd2, 0x2e, 0x41,
0xec, 0x8b, 0x40, 0xd0, 0xc5, 0x0f, 0xa0, 0x1d, 0x64, 0x5b, 0x49, 0x92, 0x8e, 0xc2, 0x58, 0xea,
0x24, 0xb5, 0x73, 0x76, 0x91, 0xc9, 0x9f, 0x41, 0x07, 0xdd, 0x0f, 0xb4, 0xd4, 0x45, 0xf7, 0xdd,
0x81, 0x06, 0xf2, 0x8a, 0xeb, 0x2c, 0x45, 0xb3, 0x8c, 0x7a, 0x79, 0x51, 0x89, 0xe0, 0x5f, 0x19,
0x0f, 0xdb, 0xc7, 0x2a, 0xd6, 0x95, 0xa6, 0x20, 0x9a, 0x1c, 0xb4, 0x85, 0x21, 0x18, 0x37, 0xa1,
0x58, 0xcc, 0x2b, 0x25, 0x66, 0xe4, 0x0a, 0x92, 0xf1, 0x9f, 0x1d, 0x80, 0x1c, 0xd0, 0x34, 0x2b,
0x4c, 0x9c, 0xcb, 0x4d, 0xd8, 0xc7, 0x95, 0x7d, 0x3c, 0xdf, 0x27, 0x85, 0x48, 0x54, 0xb6, 0x76,
0x2f, 0x6f, 0x0b, 0xdb, 0xf2, 0x9d, 0x52, 0xdf, 0xf0, 0x6d, 0x99, 0x70, 0x15, 0xb4, 0xb7, 0xa2,
0x69, 0xa6, 0x55, 0x6a, 0x11, 0xe1, 0xbb, 0x61, 0x18, 0x45, 0x7e, 0x4a, 0xc6, 0xe2, 0x14, 0xb1,
0x07, 0x50, 0x47, 0xa4, 0xa6, 0x37, 0xe7, 0xc3, 0x30, 0x42, 0x3e, 0xb0, 0xd3, 0xb1, 0xb0, 0xed,
0x18, 0x78, 0xf4, 0x95, 0x60, 0xdb, 0x85, 0x3e, 0x10, 0x3a, 0xe0, 0xee, 0x84, 0x31, 0x85, 0xe0,
0x0a, 0x3c, 0x12, 0x47, 0x9e, 0xd2, 0x4b, 0x89, 0x1c, 0x89, 0xfb, 0x71, 0xd5, 0x6c, 0x07, 0x9c,
0x87, 0xeb, 0xcc, 0x6c, 0xfe, 0xd0, 0xba, 0x95, 0x87, 0x76, 0x00, 0xab, 0x66, 0x13, 0xfc, 0x9f,
0x4e, 0x7f, 0xad, 0xc1, 0xaa, 0x50, 0x59, 0xf8, 0x4a, 0x05, 0x71, 0xa6, 0xd3, 0xe9, 0x50, 0x87,
0x49, 0x8c, 0xf6, 0x5f, 0x26, 0x07, 0x36, 0xd5, 0xae, 0x30, 0xc4, 0x9b, 0x74, 0x12, 0x7b, 0x04,
0xad, 0xd9, 0xee, 0x9f, 0x57, 0xad, 0xaa, 0xb0, 0x47, 0xb0, 0x34, 0x48, 0xa6, 0xe9, 0xb0, 0x98,
0xed, 0x3b, 0xa5, 0xb6, 0x41, 0x66, 0xc4, 0x22, 0x57, 0xab, 0xf4, 0x51, 0xfd, 0xf5, 0x7d, 0xc4,
0x9e, 0xce, 0xf4, 0x11, 0x7d, 0x8d, 0xb5, 0x36, 0xde, 0x29, 0x0d, 0x2e, 0x88, 0xc5, 0x45, 0x6d,
0xfe, 0x93, 0x03, 0x37, 0xaa, 0x10, 0xde, 0x68, 0x30, 0x8a, 0x8a, 0xd4, 0x16, 0x56, 0xc4, 0x5d,
0x54, 0x11, 0xaf, 0xac, 0x48, 0xf9, 0x76, 0xd7, 0x2b, 0x6f, 0x37, 0x3f, 0x82, 0xbb, 0x73, 0x65,
0xda, 0x4a, 0xc6, 0x13, 0xec, 0x87, 0xff, 0x50, 0x2e, 0x5c, 0x19, 0x69, 0x6a, 0x0b, 0xd5, 0x14,
0x86, 0xe0, 0x9f, 0xc2, 0xdb, 0x03, 0xa5, 0x2b, 0x45, 0xca, 0xbb, 0xad, 0x0b, 0xee, 0xae, 0x3a,
0xb9, 0x24, 0x7c, 0x14, 0xf1, 0xcf, 0xc1, 0xdf, 0x9f, 0x8c, 0xa4, 0x56, 0xd7, 0xb2, 0xde, 0x84,
0xe5, 0xbd, 0x64, 0x92, 0x44, 0xc9, 0xe1, 0xd9, 0x15, 0x23, 0xef, 0xc3, 0x92, 0xd9, 0x8f, 0xe6,
0x33, 0xb2, 0x29, 0x72, 0x92, 0xdf, 0xc2, 0x86, 0x1e, 0xca, 0x68, 0x38, 0x8d, 0x10, 0x06, 0x7e,
0x4f, 0x66, 0x9b, 0x9d, 0xdf, 0xce, 0xd7, 0x9c, 0xdf, 0xcf, 0xd7, 0x9c, 0x3f, 0xcf, 0xd7, 0x9c,
0x5f, 0xfe, 0x5a, 0x7b, 0xeb, 0xa0, 0x41, 0x7f, 0x16, 0x8f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff,
0xfa, 0xf5, 0x5b, 0x36, 0x6a, 0x0c, 0x00, 0x00,
// 1035 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1b, 0x45,
0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x8c, 0x53, 0x67, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45,
0x0d, 0x1c, 0xa2, 0x92, 0x5e, 0x78, 0x55, 0x8a, 0x12, 0xa7, 0x62, 0x11, 0x89, 0x60, 0x36, 0xe9,
0x01, 0x89, 0xc3, 0xd4, 0x1e, 0xa5, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x1e, 0x3d, 0x70, 0x85,
0x0b, 0x17, 0x4e, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f,
0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xfb, 0xbd, 0x5f, 0xbf, 0xef, 0x9b, 0x85, 0xee, 0x24, 0x8d, 0xce,
0xb9, 0x12, 0xdb, 0x93, 0x54, 0x2a, 0x49, 0x56, 0xa3, 0x44, 0x89, 0x34, 0xe1, 0x31, 0xed, 0x40,
0x3b, 0x48, 0x46, 0xe2, 0xf2, 0x50, 0x28, 0x4e, 0x7f, 0x77, 0xa0, 0xfd, 0x34, 0xe5, 0x63, 0xa1,
0x29, 0xf2, 0x3e, 0xb4, 0xf7, 0xf9, 0xf0, 0x85, 0x38, 0xbe, 0x9a, 0x08, 0xdf, 0xed, 0x3b, 0x5b,
0x6d, 0x56, 0x31, 0x4a, 0x69, 0x18, 0xbd, 0x14, 0xbe, 0xd7, 0x77, 0xb6, 0xba, 0xac, 0x62, 0x90,
0x3e, 0x74, 0x8e, 0xa3, 0xb1, 0xf8, 0x3e, 0xe7, 0x89, 0xca, 0xc7, 0x7e, 0x13, 0xad, 0xeb, 0x2c,
0xf2, 0x10, 0x5a, 0x4f, 0x23, 0x11, 0x8f, 0x32, 0x7f, 0xa5, 0xef, 0x6e, 0x75, 0x76, 0xee, 0x6c,
0x17, 0x39, 0x6d, 0x23, 0x9f, 0x59, 0x31, 0xa5, 0xb0, 0x16, 0x8c, 0x27, 0x32, 0x55, 0x4c, 0x64,
0x13, 0x99, 0x64, 0x82, 0xf4, 0xc0, 0x3d, 0x48, 0x53, 0xdf, 0x41, 0xa7, 0xfa, 0x93, 0xfe, 0x0c,
0xbd, 0xbd, 0x58, 0x0e, 0xcf, 0x06, 0x5c, 0x71, 0x26, 0x7e, 0xca, 0x45, 0xa6, 0xc8, 0x3d, 0x68,
0x62, 0x65, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x15, 0xfa, 0x0d, 0xc3, 0x45, 0x42, 0x73, 0xd1, 0x1e,
0xcb, 0xf4, 0x98, 0x21, 0x34, 0x37, 0x8c, 0xa3, 0xa1, 0x29, 0xcf, 0x63, 0x86, 0x20, 0x04, 0xbc,
0x67, 0x91, 0xb8, 0xb0, 0x35, 0xe1, 0x37, 0x0d, 0x60, 0xbd, 0x16, 0xdf, 0xa6, 0xb9, 0x01, 0x2d,
0x26, 0x2f, 0x82, 0x41, 0xe6, 0x3b, 0x7d, 0x77, 0xcb, 0x63, 0x96, 0xc2, 0xce, 0xc9, 0x38, 0x1f,
0x27, 0x5a, 0xd4, 0x40, 0x51, 0xc5, 0xa0, 0xf7, 0xa1, 0x89, 0x6d, 0xd4, 0x55, 0x56, 0xb6, 0xfa,
0x93, 0xfe, 0xe2, 0x40, 0xfb, 0x90, 0x5f, 0x62, 0x1a, 0x19, 0x79, 0x02, 0xab, 0xa1, 0xe2, 0xc9,
0x88, 0xa7, 0x23, 0x54, 0xea, 0xec, 0x7c, 0x58, 0xb5, 0xb0, 0x54, 0xdb, 0x2e, 0x74, 0x0e, 0x12,
0x95, 0x5e, 0xb1, 0xd2, 0xe4, 0xbd, 0x2f, 0xa1, 0x7b, 0x43, 0xa4, 0xe3, 0x9d, 0x89, 0xab, 0xa2,
0xab, 0x67, 0xe2, 0x4a, 0xd7, 0x7f, 0xce, 0xe3, 0xdc, 0xf4, 0xca, 0x63, 0x86, 0xf8, 0xa2, 0xf1,
0x99, 0x43, 0x77, 0x81, 0xec, 0xa7, 0x82, 0x2b, 0x81, 0x41, 0x0e, 0x45, 0x96, 0xf1, 0x53, 0xb1,
0xb8, 0xe3, 0xa6, 0x8b, 0x8d, 0x5a, 0x17, 0xe9, 0x27, 0x40, 0x06, 0x22, 0x16, 0x4a, 0x58, 0xf4,
0xbd, 0xc2, 0x03, 0x0d, 0x8b, 0x68, 0xb7, 0xeb, 0x92, 0x87, 0xe0, 0x69, 0xf0, 0x62, 0xb0, 0xce,
0xce, 0xdd, 0xaa, 0x23, 0x25, 0xca, 0x19, 0x2a, 0xd0, 0xa8, 0x70, 0x6a, 0x01, 0x7f, 0x4b, 0x09,
0x73, 0x40, 0x53, 0x84, 0x72, 0xa7, 0x43, 0x95, 0x2b, 0x64, 0x43, 0xed, 0x16, 0xb5, 0x2e, 0x1b,
0x8a, 0x9e, 0x96, 0xc9, 0xea, 0x9d, 0x58, 0x26, 0xd9, 0x8f, 0xa0, 0x89, 0xb6, 0x36, 0xdb, 0x99,
0x6d, 0x33, 0x52, 0xfa, 0xac, 0x4c, 0x75, 0xd9, 0x40, 0xf7, 0xea, 0x81, 0xda, 0x85, 0xdf, 0x1f,
0xac, 0xae, 0xde, 0x9e, 0x23, 0x6d, 0x63, 0x3c, 0xe1, 0xf7, 0xe2, 0x99, 0x4d, 0x35, 0x52, 0xfb,
0xd6, 0xeb, 0x96, 0xf9, 0x6e, 0xdf, 0xd5, 0xbe, 0x91, 0xa0, 0x8f, 0xa1, 0x15, 0x0e, 0x5f, 0x88,
0x31, 0x27, 0x1f, 0xc3, 0x0a, 0xa6, 0x26, 0x32, 0xbb, 0x11, 0x77, 0xa6, 0xe6, 0xcf, 0x0a, 0x39,
0x1d, 0xd8, 0x92, 0x16, 0x24, 0xd4, 0xc2, 0xd0, 0x99, 0xef, 0xcd, 0xdc, 0x26, 0xcd, 0x67, 0x56,
0x4c, 0x0f, 0xc0, 0x3d, 0x61, 0x81, 0xde, 0x74, 0xcc, 0xa0, 0xf0, 0x62, 0x29, 0xed, 0xfb, 0x6b,
0x99, 0x29, 0xdb, 0x20, 0xfc, 0xd6, 0xbc, 0xef, 0x64, 0xaa, 0xb0, 0x3d, 0x5d, 0x86, 0xdf, 0xf4,
0x47, 0xf0, 0x8e, 0xe4, 0x48, 0x90, 0x35, 0x68, 0x04, 0x03, 0xeb, 0xa3, 0x11, 0x0c, 0xc8, 0x07,
0xe8, 0xde, 0xf6, 0xa5, 0x5b, 0x25, 0x71, 0xc2, 0x02, 0x86, 0x81, 0x1f, 0x40, 0x37, 0xc8, 0xf6,
0xa5, 0x4c, 0x47, 0x51, 0xc2, 0x95, 0x4c, 0xd1, 0xeb, 0x2a, 0xbb, 0xc9, 0xa4, 0xbb, 0xd0, 0xd3,
0xee, 0x43, 0xc5, 0x55, 0x89, 0xbe, 0x0d, 0x68, 0x69, 0x5e, 0x19, 0xce, 0x52, 0xb8, 0xad, 0x5a,
0xaf, 0x18, 0x2a, 0x12, 0xf4, 0x5b, 0xe3, 0xe1, 0xe0, 0x5c, 0x24, 0xaa, 0x06, 0x0a, 0xa4, 0xd1,
0x41, 0x97, 0x19, 0x82, 0x50, 0x53, 0x8a, 0xcd, 0x79, 0xad, 0xca, 0x59, 0x73, 0x19, 0xca, 0xe8,
0x6f, 0x0e, 0x40, 0x91, 0x50, 0x9e, 0x95, 0x26, 0xce, 0x62, 0x13, 0xf2, 0x69, 0xed, 0xf2, 0xcd,
0xe2, 0xa4, 0x14, 0xb1, 0xda, 0x7d, 0xdc, 0x2a, 0x60, 0x61, 0x21, 0xdf, 0xab, 0xf4, 0x0d, 0xdf,
0x8e, 0x49, 0x9f, 0x82, 0xee, 0x7e, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0xa1, 0x0d, 0xa3,
0xec, 0x4f, 0xc5, 0x98, 0xdf, 0x22, 0xf2, 0x00, 0x9a, 0x3a, 0x53, 0x83, 0xcd, 0xd9, 0x32, 0x8c,
0x90, 0x86, 0x76, 0x3b, 0xe6, 0xc2, 0x8e, 0x80, 0x87, 0x6f, 0xad, 0x85, 0x0b, 0x3e, 0xb3, 0x3d,
0x70, 0x0f, 0xa3, 0x04, 0x4b, 0x70, 0x99, 0xfe, 0x44, 0x0e, 0xbf, 0xc4, 0x37, 0x49, 0x73, 0xb8,
0xbe, 0x8f, 0xeb, 0xe6, 0x3a, 0xe8, 0x7d, 0x58, 0x66, 0x67, 0x8b, 0x27, 0xcd, 0xad, 0x3d, 0x69,
0x21, 0xac, 0x9b, 0x4b, 0xf0, 0x26, 0x9d, 0xfe, 0xd9, 0x80, 0x75, 0x26, 0xb2, 0xe8, 0xa5, 0x08,
0x92, 0x4c, 0xa5, 0xf9, 0x50, 0x45, 0x32, 0xd1, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xd5, 0x2e, 0x33,
0xc4, 0xeb, 0x20, 0x89, 0x3c, 0x82, 0xce, 0x34, 0xfa, 0x67, 0x55, 0xeb, 0x2a, 0xe4, 0x11, 0xac,
0x84, 0x32, 0x4f, 0x87, 0xe5, 0x6e, 0x6f, 0x54, 0xda, 0x26, 0x33, 0x23, 0x66, 0x85, 0x5a, 0x0d,
0x47, 0xcd, 0x57, 0xe3, 0x88, 0x3c, 0x99, 0xc2, 0x91, 0xdf, 0x42, 0x83, 0x77, 0x2b, 0x83, 0x1b,
0x62, 0x76, 0x53, 0x9b, 0xfe, 0xea, 0xc0, 0xdb, 0xf5, 0x14, 0x5e, 0x6b, 0x31, 0xca, 0x89, 0x34,
0xe6, 0x4e, 0xc4, 0x9d, 0x37, 0x11, 0xaf, 0x9a, 0x48, 0xf5, 0x3a, 0x37, 0xeb, 0xaf, 0xf3, 0x19,
0xdc, 0x9f, 0x19, 0xd3, 0xbe, 0x1c, 0x4f, 0x34, 0x1e, 0xfe, 0xc7, 0xb8, 0xf4, 0xc9, 0x48, 0x53,
0x3b, 0xa8, 0x36, 0x33, 0x04, 0xfd, 0x1c, 0xde, 0x09, 0x85, 0xaa, 0x0d, 0xa9, 0x40, 0x5b, 0x1f,
0xdc, 0x23, 0x71, 0xb1, 0xa0, 0x7c, 0x2d, 0xa2, 0x5f, 0x81, 0x7f, 0x32, 0x19, 0x71, 0x25, 0x96,
0xb2, 0xde, 0x83, 0xd5, 0x63, 0x39, 0x91, 0xb1, 0x3c, 0xbd, 0xba, 0x65, 0xe5, 0x7d, 0x58, 0x31,
0xf7, 0xd1, 0xfc, 0xb0, 0xb5, 0x59, 0x41, 0xd2, 0xbb, 0x1a, 0xd0, 0x43, 0x1e, 0x0f, 0xf3, 0x58,
0xa7, 0xa1, 0xff, 0xdc, 0xb2, 0xbd, 0xde, 0x5f, 0xd7, 0x9b, 0xce, 0xdf, 0xd7, 0x9b, 0xce, 0x3f,
0xd7, 0x9b, 0xce, 0x1f, 0xff, 0x6e, 0xbe, 0xf5, 0xbc, 0x85, 0xff, 0xdd, 0x8f, 0xff, 0x0b, 0x00,
0x00, 0xff, 0xff, 0xd3, 0x15, 0x68, 0xea, 0x88, 0x0b, 0x00, 0x00,
}

6
internal/private.proto
View file

@ -6,12 +6,10 @@ message IndexMeta {
}
message FrameMeta {
bool InverseEnabled = 2;
string CacheType = 3;
uint32 CacheSize = 4;
string TimeQuantum = 5;
bool RangeEnabled = 6;
repeated Field Fields = 7;
repeated Field Fields = 7;
}
message ImportResponse {
@ -37,13 +35,11 @@ message Cache {
message MaxSlices {
map<string, uint64> Standard = 1;
map<string, uint64> Inverse = 2;
}
message CreateSliceMessage {
string Index = 1;
uint64 Slice = 2;
bool IsInverse = 3;
}
message DeleteIndexMessage {

158
internal/public.pb.go
View file

@ -43,9 +43,9 @@ var _ = math.Inf
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
type Row struct {
Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"`
Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"`
Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"`
Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"`
}
func (m *Row) Reset() { *m = Row{} }
@ -53,9 +53,9 @@ func (m *Row) String() string { return proto.CompactTextString(m) }
func (*Row) ProtoMessage() {}
func (*Row) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{0} }
func (m *Row) GetBits() []uint64 {
func (m *Row) GetColumns() []uint64 {
if m != nil {
return m.Bits
return m.Columns
}
return nil
}
@ -267,12 +267,12 @@ func (m *AttrMap) GetAttrs() []*Attr {
}
type QueryRequest struct {
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
ExcludeAttrs bool `protobuf:"varint,6,opt,name=ExcludeAttrs,proto3" json:"ExcludeAttrs,omitempty"`
ExcludeBits bool `protobuf:"varint,7,opt,name=ExcludeBits,proto3" json:"ExcludeBits,omitempty"`
Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"`
Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"`
ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"`
Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"`
ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"`
ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"`
}
func (m *QueryRequest) Reset() { *m = QueryRequest{} }
@ -308,16 +308,16 @@ func (m *QueryRequest) GetRemote() bool {
return false
}
func (m *QueryRequest) GetExcludeAttrs() bool {
func (m *QueryRequest) GetExcludeRowAttrs() bool {
if m != nil {
return m.ExcludeAttrs
return m.ExcludeRowAttrs
}
return false
}
func (m *QueryRequest) GetExcludeBits() bool {
func (m *QueryRequest) GetExcludeColumns() bool {
if m != nil {
return m.ExcludeBits
return m.ExcludeColumns
}
return false
}
@ -575,10 +575,10 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if len(m.Bits) > 0 {
dAtA2 := make([]byte, len(m.Bits)*10)
if len(m.Columns) > 0 {
dAtA2 := make([]byte, len(m.Columns)*10)
var j1 int
for _, num := range m.Bits {
for _, num := range m.Columns {
for num >= 1<<7 {
dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80)
num >>= 7
@ -902,20 +902,20 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) {
}
i++
}
if m.ExcludeAttrs {
if m.ExcludeRowAttrs {
dAtA[i] = 0x30
i++
if m.ExcludeAttrs {
if m.ExcludeRowAttrs {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if m.ExcludeBits {
if m.ExcludeColumns {
dAtA[i] = 0x38
i++
if m.ExcludeBits {
if m.ExcludeColumns {
dAtA[i] = 1
} else {
dAtA[i] = 0
@ -1263,9 +1263,9 @@ func encodeVarintPublic(dAtA []byte, offset int, v uint64) int {
func (m *Row) Size() (n int) {
var l int
_ = l
if len(m.Bits) > 0 {
if len(m.Columns) > 0 {
l = 0
for _, e := range m.Bits {
for _, e := range m.Columns {
l += sovPublic(uint64(e))
}
n += 1 + sovPublic(uint64(l)) + l
@ -1405,10 +1405,10 @@ func (m *QueryRequest) Size() (n int) {
if m.Remote {
n += 2
}
if m.ExcludeAttrs {
if m.ExcludeRowAttrs {
n += 2
}
if m.ExcludeBits {
if m.ExcludeColumns {
n += 2
}
return n
@ -1615,7 +1615,7 @@ func (m *Row) Unmarshal(dAtA []byte) error {
break
}
}
m.Bits = append(m.Bits, v)
m.Columns = append(m.Columns, v)
} else if wireType == 2 {
var packedLen int
for shift := uint(0); ; shift += 7 {
@ -1655,10 +1655,10 @@ func (m *Row) Unmarshal(dAtA []byte) error {
break
}
}
m.Bits = append(m.Bits, v)
m.Columns = append(m.Columns, v)
}
} else {
return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field Columns", wireType)
}
case 2:
if wireType != 2 {
@ -2602,7 +2602,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
m.Remote = bool(v != 0)
case 6:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeAttrs", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeRowAttrs", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
@ -2619,10 +2619,10 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.ExcludeAttrs = bool(v != 0)
m.ExcludeRowAttrs = bool(v != 0)
case 7:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeBits", wireType)
return fmt.Errorf("proto: wrong wireType = %d for field ExcludeColumns", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
@ -2639,7 +2639,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
break
}
}
m.ExcludeBits = bool(v != 0)
m.ExcludeColumns = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPublic(dAtA[iNdEx:])
@ -3795,50 +3795,50 @@ var (
func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) }
var fileDescriptorPublic = []byte{
// 707 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a,
0x14, 0xbe, 0x13, 0x3b, 0x7f, 0x27, 0x49, 0x55, 0x8d, 0xee, 0xed, 0xb5, 0xae, 0xae, 0x82, 0x65,
0xb1, 0xf0, 0x2a, 0x95, 0xc2, 0x1e, 0x44, 0xfa, 0x23, 0x45, 0x15, 0x15, 0x4c, 0x4a, 0x59, 0xbb,
0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x08, 0x16,
0x88, 0x15, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x73, 0xc6, 0x13, 0x3b, 0xad, 0x04, 0x2c, 0xd8,
0xcd, 0xf7, 0x7d, 0x33, 0xc7, 0xf3, 0xcd, 0xf9, 0x4e, 0x02, 0xc3, 0xac, 0xbc, 0x48, 0xe2, 0xcb,
0x49, 0x96, 0x2b, 0xad, 0x78, 0x2f, 0x4e, 0xb5, 0xcc, 0xd3, 0x28, 0x09, 0x16, 0xe0, 0x08, 0xb5,
0xe2, 0x1c, 0xdc, 0x59, 0xac, 0x0b, 0x8f, 0xf9, 0x4e, 0xe8, 0x0a, 0x5a, 0xf3, 0x87, 0xd0, 0x7e,
0xaa, 0x75, 0x5e, 0x78, 0x2d, 0xdf, 0x09, 0x07, 0xd3, 0x9d, 0x89, 0x3d, 0x34, 0x41, 0x5a, 0x18,
0x11, 0x4f, 0x9e, 0xc8, 0x75, 0xe1, 0x39, 0xbe, 0x13, 0xf6, 0x05, 0xad, 0x83, 0xc7, 0xe0, 0x3e,
0x8f, 0xe2, 0x9c, 0xef, 0x40, 0x6b, 0x7e, 0xe8, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd, 0x0f, 0xf9,
0xdf, 0xd0, 0x3e, 0x50, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x5d, 0x70, 0x4e, 0xe4, 0xda,
0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0xce, 0xa3, 0x64, 0xa3, 0x9e, 0x47, 0x09,
0x15, 0x71, 0x04, 0x2e, 0xb7, 0xab, 0x38, 0x55, 0x95, 0xe0, 0x25, 0x38, 0xb3, 0x58, 0xa3, 0x28,
0xd4, 0x6a, 0xf3, 0x55, 0x03, 0xf8, 0x7f, 0xd0, 0x3b, 0x50, 0x49, 0xb9, 0x4c, 0xe7, 0x87, 0xd5,
0xb7, 0x37, 0x98, 0xff, 0x0f, 0xfd, 0xb3, 0x78, 0x29, 0x0b, 0x1d, 0x2d, 0x33, 0xba, 0x84, 0x23,
0x6a, 0x22, 0x78, 0x05, 0x23, 0xb3, 0x13, 0xdd, 0x2e, 0xa4, 0xbe, 0xe7, 0xe9, 0xf7, 0x5e, 0xe9,
0xbe, 0xc7, 0xf7, 0x0c, 0x5c, 0xd4, 0xac, 0xc4, 0x36, 0x12, 0x3e, 0xe9, 0xd9, 0x3a, 0x93, 0xd5,
0x4d, 0x69, 0xcd, 0x7d, 0x18, 0x2c, 0x74, 0x1e, 0xa7, 0xd7, 0xe7, 0x51, 0x52, 0xca, 0xaa, 0x50,
0x93, 0x42, 0x8f, 0xf3, 0x54, 0x1b, 0xd9, 0x25, 0x1b, 0x1b, 0x8c, 0x1e, 0x67, 0x4a, 0x25, 0x46,
0x6c, 0xfb, 0x2c, 0xec, 0x89, 0x9a, 0xe0, 0x63, 0x80, 0xe3, 0x44, 0x45, 0xd5, 0xd9, 0x8e, 0xcf,
0x42, 0x26, 0x1a, 0x4c, 0xb0, 0x0f, 0x5d, 0xbc, 0xe9, 0xb3, 0x28, 0xab, 0xdd, 0xb2, 0x9f, 0xb8,
0x0d, 0x3e, 0x30, 0x18, 0xbe, 0x28, 0x65, 0xbe, 0x16, 0xf2, 0x4d, 0x29, 0x0b, 0xea, 0x0a, 0xe1,
0xca, 0xa5, 0x01, 0x7c, 0x0f, 0x3a, 0x8b, 0x24, 0xbe, 0x94, 0xe6, 0xed, 0x5c, 0x51, 0x21, 0xf4,
0x5a, 0xbf, 0x79, 0x41, 0x5e, 0x7b, 0xa2, 0x49, 0xe1, 0x49, 0x21, 0x97, 0x4a, 0x5b, 0x33, 0x15,
0xe2, 0x01, 0x0c, 0x8f, 0x6e, 0x2e, 0x93, 0xf2, 0x4a, 0x9a, 0xa3, 0x1d, 0x52, 0xb7, 0x38, 0xac,
0x5e, 0x61, 0x4a, 0x7c, 0xd7, 0x54, 0x6f, 0x50, 0xc1, 0x5b, 0x06, 0xa3, 0xea, 0xfa, 0x45, 0xa6,
0xd2, 0x42, 0x62, 0x8f, 0x8e, 0xf2, 0xdc, 0xf6, 0xe8, 0x28, 0xcf, 0xf9, 0x3e, 0x74, 0x85, 0x2c,
0xca, 0x44, 0xdb, 0xc6, 0xff, 0x53, 0x3f, 0x85, 0x3d, 0x5b, 0x26, 0x5a, 0xd8, 0x5d, 0xfc, 0x09,
0xec, 0x6c, 0x05, 0xc9, 0x4c, 0xcc, 0x60, 0xfa, 0x6f, 0x7d, 0x6e, 0x4b, 0x17, 0x77, 0xb6, 0x07,
0x1f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x03, 0x9a, 0x5c, 0xba, 0xd3, 0x60, 0x3a, 0xaa, 0xab, 0x08,
0xb5, 0x12, 0x34, 0xd3, 0x43, 0x60, 0xa7, 0x55, 0x86, 0xd8, 0x29, 0x76, 0x0e, 0x67, 0xd2, 0x7e,
0xb6, 0xd1, 0x39, 0xa4, 0x85, 0x11, 0xb9, 0x07, 0xdd, 0x83, 0xd7, 0x51, 0x7a, 0x2d, 0xaf, 0x28,
0x43, 0x3d, 0x61, 0x21, 0x9f, 0xd4, 0x33, 0x49, 0x8f, 0x3e, 0x98, 0xf2, 0xba, 0x84, 0x55, 0x44,
0x3d, 0xb7, 0x36, 0xc4, 0xd8, 0x82, 0x91, 0x09, 0x71, 0xf0, 0x8d, 0xc1, 0x68, 0xbe, 0xcc, 0x54,
0xae, 0x1b, 0xc1, 0x98, 0xa7, 0x57, 0xf2, 0xc6, 0x06, 0x83, 0x00, 0xb2, 0xc7, 0x79, 0xb4, 0x34,
0x13, 0xd0, 0x17, 0x06, 0x20, 0x4b, 0x01, 0xa1, 0x40, 0xb8, 0xc2, 0x00, 0x8a, 0x02, 0xce, 0x78,
0xe1, 0xb9, 0x26, 0x44, 0x06, 0x61, 0xe4, 0xed, 0x88, 0x17, 0x5e, 0x9b, 0xa4, 0x9a, 0xc0, 0xc8,
0x6f, 0x66, 0x1c, 0x63, 0xe2, 0x84, 0x8e, 0x68, 0x30, 0xf8, 0x0e, 0x42, 0xad, 0xe8, 0x87, 0xad,
0x4b, 0x3f, 0x6c, 0x16, 0xe2, 0x49, 0x53, 0x86, 0xc4, 0x1e, 0x89, 0x0d, 0x26, 0xf8, 0xc4, 0x80,
0x1b, 0x8f, 0x34, 0x3c, 0x7f, 0xce, 0x28, 0xee, 0x8d, 0x65, 0x62, 0x1a, 0x83, 0x7b, 0x11, 0xfc,
0xc2, 0xe6, 0x1e, 0x74, 0xe8, 0x16, 0xd6, 0x62, 0x85, 0xee, 0x98, 0xe8, 0xde, 0x35, 0x31, 0xdb,
0xfd, 0x7c, 0x3b, 0x66, 0x5f, 0x6e, 0xc7, 0xec, 0xeb, 0xed, 0x98, 0xbd, 0xfb, 0x3e, 0xfe, 0xeb,
0xa2, 0x43, 0x7f, 0x1c, 0x8f, 0x7e, 0x04, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x35, 0x66, 0x28, 0x48,
0x06, 0x00, 0x00,
// 709 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c,
0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c,
0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba,
0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4,
0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd,
0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64,
0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8,
0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82,
0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69,
0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f,
0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd,
0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e,
0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e,
0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62,
0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf,
0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8,
0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77,
0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e,
0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35,
0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88,
0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59,
0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7,
0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c,
0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd,
0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85,
0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3,
0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18,
0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3,
0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4,
0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96,
0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60,
0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec,
0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd,
0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e,
0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c,
0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41,
0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62,
0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9,
0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b,
0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82,
0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8,
0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd,
0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7,
0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7,
0xa4, 0x5a, 0x06, 0x00, 0x00,
}

6
internal/public.proto
View file

@ -3,7 +3,7 @@ syntax = "proto3";
package internal;
message Row {
repeated uint64 Bits = 1;
repeated uint64 Columns = 1;
repeated string Keys = 3;
repeated Attr Attrs = 2;
}
@ -49,8 +49,8 @@ message QueryRequest {
repeated uint64 Slices = 2;
bool ColumnAttrs = 3;
bool Remote = 5;
bool ExcludeAttrs = 6;
bool ExcludeBits = 7;
bool ExcludeRowAttrs = 6;
bool ExcludeColumns = 7;
}
message QueryResponse {

7
pilosa.go
View file

@ -32,10 +32,9 @@ var (
ErrIndexNotFound = errors.New("index not found")
// ErrFrameRequired is returned when no frame is specified.
ErrFrameRequired = errors.New("frame required")
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
ErrFrameInverseDisabled = errors.New("frame inverse disabled")
ErrFrameRequired = errors.New("frame required")
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")

28
pql/ast.go
View file

@ -177,34 +177,6 @@ func (c *Call) String() string {
return buf.String()
}
// SupportsInverse indicates that the call may be on an inverse frame.
func (c *Call) SupportsInverse() bool {
return c.Name == "Bitmap" || c.Name == "TopN"
}
// IsInverse specifies if the call is for an inverse view.
// Return defaults to false unless absolutely sure of inversion.
func (c *Call) IsInverse(rowLabel, columnLabel string) bool {
if c.SupportsInverse() {
// Top-n has an explicit inverse flag.
if c.Name == "TopN" {
inverse, _ := c.Args["inverse"].(bool)
return inverse
}
// Bitmap calls use the row/column labels to determine whether inverse.
_, rowOK, rowErr := c.UintArg(rowLabel)
_, columnOK, columnErr := c.UintArg(columnLabel)
if rowErr != nil || columnErr != nil {
return false
}
if !rowOK && columnOK {
return true
}
}
return false
}
// HasConditionArg returns true if any arg is a conditional.
func (c *Call) HasConditionArg() bool {
for _, v := range c.Args {

66
pql/ast_test.go
View file

@ -67,69 +67,3 @@ func TestCondition_Value(t *testing.T) {
}
})
}
// Ensure call can be converted into a string.
func TestCall_SupportsInverse(t *testing.T) {
t.Run("Bitmap", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap()`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].SupportsInverse() != true {
t.Fatalf("call should support inverse: %s", q.Calls[0])
}
})
t.Run("Count Bitmap", func(t *testing.T) {
q, err := pql.ParseString(`Count(Bitmap())`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].SupportsInverse() == true {
t.Fatalf("call should not support inverse: %s", q.Calls[0])
}
})
t.Run("Union Bitmaps", func(t *testing.T) {
q, err := pql.ParseString(`Union(Bitmap(), Bitmap())`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].SupportsInverse() == true {
t.Fatalf("call should not support inverse: %s", q.Calls[0])
}
})
}
// Ensure call is correctly determined to be against an inverse view.
func TestCall_IsInverse(t *testing.T) {
t.Run("Bitmap Row", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap(frame="f", row=1)`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("row", "col") != false {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
t.Run("Bitmap Column", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap(frame="f", col=1)`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("row", "col") != true {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
t.Run("Bitmap Column No Label", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap(frame="f", col=1)`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("rowX", "colX") != false {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
t.Run("Count", func(t *testing.T) {
q, err := pql.ParseString(`Count(Bitmap(frame="f", col=1))`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("row", "col") != false {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
}

18
roaring/roaring.go
View file

@ -1461,22 +1461,14 @@ func (c *Container) arrayMax() uint16 {
func (c *Container) bitmapMax() uint16 {
// Search bitmap in reverse order.
for i := len(c.bitmap) - 1; i >= 0; i-- {
for i := len(c.bitmap); i > 0; i-- {
// If value is zero then skip.
v := c.bitmap[i]
if v == 0 {
continue
v := c.bitmap[i-1]
if v != 0 {
r := bits.LeadingZeros64(v)
return uint16((i-1)*64 + 63 - r)
}
// Find the highest set bit.
for j := uint16(63); ; j-- {
if v&(1<<j) != 0 {
return uint16(i)*64 + j
}
if j == 0 {
break
}
}
}
return 0
}

41
row.go
View file

@ -22,7 +22,8 @@ import (
"github.com/pilosa/pilosa/roaring"
)
// Row represents a set of bits.
// Row is a set of integers (the associated columns), and attributes which are
// arbitrary key/value pairs storing metadata about what the row represents.
type Row struct {
segments []RowSegment
@ -31,9 +32,9 @@ type Row struct {
}
// NewRow returns a new instance of Row.
func NewRow(bits ...uint64) *Row {
func NewRow(columns ...uint64) *Row {
r := &Row{}
for _, i := range bits {
for _, i := range columns {
r.SetBit(i)
}
return r
@ -151,12 +152,12 @@ func (r *Row) Difference(other *Row) *Row {
return &Row{segments: segments}
}
// SetBit sets the i-th bit of the row.
// SetBit sets the i-th column of the row.
func (r *Row) SetBit(i uint64) (changed bool) {
return r.createSegmentIfNotExists(i / SliceWidth).SetBit(i)
}
// ClearBit clears the i-th bit of the row.
// ClearBit clears the i-th column of the row.
func (r *Row) ClearBit(i uint64) (changed bool) {
s := r.segment(i / SliceWidth)
if s == nil {
@ -226,7 +227,7 @@ func (r *Row) DecrementCount(i uint64) {
}
}
// Count returns the number of set bits in the row.
// Count returns the number of columns in the row.
func (r *Row) Count() uint64 {
var n uint64
for i := range r.segments {
@ -238,10 +239,10 @@ func (r *Row) Count() uint64 {
// MarshalJSON returns a JSON-encoded byte slice of r.
func (r *Row) MarshalJSON() ([]byte, error) {
var o struct {
Attrs map[string]interface{} `json:"attrs"`
Bits []uint64 `json:"bits"`
Attrs map[string]interface{} `json:"attrs"`
Columns []uint64 `json:"columns"`
}
o.Bits = r.Bits()
o.Columns = r.Columns()
o.Attrs = r.Attrs
if o.Attrs == nil {
@ -251,11 +252,11 @@ func (r *Row) MarshalJSON() ([]byte, error) {
return json.Marshal(&o)
}
// Bits returns the bits in r as a slice of ints.
func (r *Row) Bits() []uint64 {
// Columns returns the columns in r as a slice of ints.
func (r *Row) Columns() []uint64 {
a := make([]uint64, 0, r.Count())
for i := range r.segments {
a = append(a, r.segments[i].Bits()...)
a = append(a, r.segments[i].Columns()...)
}
return a
}
@ -267,8 +268,8 @@ func encodeRow(r *Row) *internal.Row {
}
return &internal.Row{
Bits: r.Bits(),
Attrs: encodeAttrs(r.Attrs),
Columns: r.Columns(),
Attrs: encodeAttrs(r.Attrs),
}
}
@ -280,7 +281,7 @@ func decodeRow(pr *internal.Row) *Row {
r := NewRow()
r.Attrs = decodeAttrs(pr.Attrs)
for _, v := range pr.Bits {
for _, v := range pr.Columns {
r.SetBit(v)
}
return r
@ -372,7 +373,7 @@ func (s *RowSegment) Xor(other *RowSegment) *RowSegment {
}
}
// SetBit sets the i-th bit of the row.
// SetBit sets the i-th column of the row.
func (s *RowSegment) SetBit(i uint64) (changed bool) {
s.ensureWritable()
changed, _ = s.data.Add(i)
@ -382,7 +383,7 @@ func (s *RowSegment) SetBit(i uint64) (changed bool) {
return changed
}
// ClearBit clears the i-th bit of the row.
// ClearBit clears the i-th column of the row.
func (s *RowSegment) ClearBit(i uint64) (changed bool) {
s.ensureWritable()
@ -398,8 +399,8 @@ func (s *RowSegment) InvalidateCount() {
s.n = s.data.Count()
}
// Bits returns a list of all bits set in the segment.
func (s *RowSegment) Bits() []uint64 {
// Columns returns a list of all columns set in the segment.
func (s *RowSegment) Columns() []uint64 {
a := make([]uint64, 0, s.Count())
itr := s.data.Iterator()
for v, eof := itr.Next(); !eof; v, eof = itr.Next() {
@ -408,7 +409,7 @@ func (s *RowSegment) Bits() []uint64 {
return a
}
// Count returns the number of set bits in the row.
// Count returns the number of set columns in the row.
func (s *RowSegment) Count() uint64 { return s.n }
// ensureWritable clones the segment if it is pointing to non-writable data.

22
row_test.go
View file

@ -47,7 +47,7 @@ func TestRow_Merge(t *testing.T) {
if cnt := test.r1.Count(); cnt != test.exp {
t.Fatalf("merged count %d is not %d", cnt, test.exp)
}
if length := len(test.r1.Bits()); uint64(length) != test.exp {
if length := len(test.r1.Columns()); uint64(length) != test.exp {
t.Fatalf("merged length %d is not %d", length, test.exp)
}
})
@ -65,15 +65,15 @@ func TestRow_Xor(t *testing.T) {
t.Fatalf("Test 1 Count after xor %d != 3\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Results %v != expected %v\n", res.Columns(), exp)
}
res = r2.Xor(r1)
if res.Count() != 3 {
t.Fatalf("Test 3 Count after xor %d != 3\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 4 Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 4 Results %v != expected %v\n", res.Columns(), exp)
}
}
@ -86,15 +86,15 @@ func TestRow_Union_Segment(t *testing.T) {
if res.Count() != 4 {
t.Fatalf("Test 1 Count after Union %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Columns(), exp)
}
res = r2.Union(r1)
if res.Count() != 4 {
t.Fatalf("Test 3 Count after xor %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Union Results %v != expected %v\n", res.Columns(), exp)
}
}
@ -107,7 +107,7 @@ func TestRow_Difference_Segment(t *testing.T) {
if res.Count() != 2 {
t.Fatalf("Test 1 Count after Difference %d != 5\n", res.Count())
}
if !reflect.DeepEqual(res.Bits(), exp) {
t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Bits(), exp)
if !reflect.DeepEqual(res.Columns(), exp) {
t.Fatalf("Test 2 Difference Results %v != expected %v\n", res.Columns(), exp)
}
}

26
server.go
View file

@ -440,11 +440,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
if obj.IsInverse {
idx.SetRemoteMaxInverseSlice(obj.Slice)
} else {
idx.SetRemoteMaxSlice(obj.Slice)
}
idx.SetRemoteMaxSlice(obj.Slice)
case *internal.CreateIndexMessage:
opt := IndexOptions{}
_, err := s.Holder.CreateIndex(obj.Index, opt)
@ -569,7 +565,7 @@ func (s *Server) SendTo(to *Node, pb proto.Message) error {
// where a node fails to receive a Broadcast message, or
// when a new (empty) node needs to get in sync with the
// rest of the cluster, two things are shared via gossip:
// - MaxSlice/MaxInverseSlice by Index
// - MaxSlice by Index
// - Schema
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalStatus() (proto.Message, error) {
@ -625,7 +621,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
return errors.Wrap(err, "applying schema")
}
// Sync maxSlices (standard).
// Sync maxSlices.
oldmaxslices := s.Holder.MaxSlices()
for index, newMax := range ns.MaxSlices.Standard {
localIndex := s.Holder.Index(index)
@ -641,22 +637,6 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
}
}
// Sync maxSlices (inverse).
oldMaxInverseSlices := s.Holder.MaxInverseSlices()
for index, newMaxInverse := range ns.MaxSlices.Inverse {
localIndex := s.Holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.logger.Printf("Local Index not found: %s", index)
continue
}
if newMaxInverse > oldMaxInverseSlices[index] {
oldMaxInverseSlices[index] = newMaxInverse
localIndex.SetRemoteMaxInverseSlice(newMaxInverse)
}
}
return nil
}

6
server/cluster_test.go
View file

@ -464,12 +464,12 @@ func TestClusterResize_RemoveNode(t *testing.T) {
// This is an attempt to ensure there is data on both nodes, but is not guaranteed.
// TODO: Deterministic node IDs would ensure consistent results
setBits := ""
setColumns := ""
for i := 0; i < 20; i++ {
setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth)
setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth)
}
if _, err := m0.Query("i", "", setBits); err != nil {
if _, err := m0.Query("i", "", setColumns); err != nil {
t.Fatal(err)
}

150
server/server_test.go
View file

@ -26,7 +26,6 @@ import (
"strings"
"testing"
"testing/quick"
"time"
"github.com/BurntSushi/toml"
"github.com/pilosa/pilosa"
@ -69,8 +68,8 @@ func TestMain_Set_Quick(t *testing.T) {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"bits": columnIDs,
"attrs": map[string]interface{}{},
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
@ -92,8 +91,8 @@ func TestMain_Set_Quick(t *testing.T) {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"bits": columnIDs,
"attrs": map[string]interface{}{},
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
@ -132,7 +131,7 @@ func TestMain_SetRowAttrs(t *testing.T) {
t.Fatal(err)
}
// Set bits on different rows in different frames.
// Set columns on different rows in different frames.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil {
@ -157,14 +156,14 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Query row x/1.
if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -175,19 +174,19 @@ func TestMain_SetRowAttrs(t *testing.T) {
// Query rows after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
// Query row x/2.
if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" {
} else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
@ -205,7 +204,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
t.Fatal(err)
}
// Set bits on row.
// Set columns on row.
if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil {
t.Fatal(err)
} else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil {
@ -220,7 +219,7 @@ func TestMain_SetColumnAttrs(t *testing.T) {
// Query row.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
@ -231,128 +230,11 @@ func TestMain_SetColumnAttrs(t *testing.T) {
// Query row after reopening.
if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil {
t.Fatal(err)
} else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
} else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" {
t.Fatalf("unexpected result(reopen): %s", res)
}
}
// Ensure inverse slices get handled correctly in a multi-node query.
func TestMain_InverseSlices(t *testing.T) {
mains := test.MustRunMainWithCluster(t, 2)
m0 := mains[0]
m1 := mains[1]
// Make sure to use node0 in the cluster.
var m *test.Main
if m0.Server.NodeID < m1.Server.NodeID {
m = m0
} else {
m = m1
}
// Create frames.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal("create frame:", err)
}
// Write data on cluster.
if _, err := m.Query("i", "", fmt.Sprintf(`
SetBit(col=1, frame="f", row=1000)
SetBit(col=1, frame="f", row=2000)
SetBit(col=1, frame="f", row=%d)
`, 1*pilosa.SliceWidth)); err != nil {
t.Fatal("setting bits:", err)
}
time.Sleep(1 * time.Second)
// Query the cluster.
if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != fmt.Sprintf(`{"results":[{"attrs":{},"bits":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
// Ensure program can set bits on one cluster and then restore to a second cluster.
func TestMain_FrameRestore(t *testing.T) {
mains1 := test.MustRunMainWithCluster(t, 2)
m10 := mains1[0]
m11 := mains1[1]
// Create frames.
client := m10.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal("create frame:", err)
}
// Write data on first cluster.
if _, err := m10.Query("i", "", `
SetBit(row=1, frame="f", col=100)
SetBit(row=1, frame="f", col=1000)
SetBit(row=1, frame="f", col=100000)
SetBit(row=1, frame="f", col=200000)
SetBit(row=1, frame="f", col=400000)
SetBit(row=1, frame="f", col=600000)
SetBit(row=1, frame="f", col=800000)
`); err != nil {
t.Fatal("setting bits:", err)
}
// Query row on first cluster.
if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Start second cluster.
mains2 := test.MustRunMainWithCluster(t, 2)
m20 := mains2[0]
defer m20.Close()
m21 := mains2[1]
defer m21.Close()
// Import from first cluster.
client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
if err := m20.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create new index:", err)
}
if err := m20.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal("create new frame:", err)
}
if err := client20.RestoreFrame(context.Background(), m10.Server.URI.HostPort(), "i", "f"); err != nil {
t.Fatal("restore frame:", err)
}
if err := client21.RestoreFrame(context.Background(), m11.Server.URI.HostPort(), "i", "f"); err != nil {
t.Fatal("restore frame:", err)
}
// Query row on second cluster.
if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("2unexpected result: %s", res)
}
}
// Ensure the host can be parsed.
func TestConfig_Parse_Host(t *testing.T) {
if c, err := ParseConfig(`bind = "local"`); err != nil {
@ -408,7 +290,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
t.Fatal("create frame:", err)
}
// Set some bits
// Set some columns
data := []string{}
for rowID := 1; rowID < 10; rowID++ {
for columnID := 1; columnID < 100; columnID++ {
@ -416,7 +298,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
}
}
if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil {
t.Fatal("setting bits:", err)
t.Fatal("setting columns:", err)
}
// Calculate caches on the first node
@ -440,7 +322,7 @@ func TestMain_RecalculateHashes(t *testing.T) {
}
}
// SetCommand represents a command to set a bit.
// SetCommand represents a command to set a column.
type SetCommand struct {
ID uint64
Frame string

6
stats_test.go
View file

@ -146,7 +146,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
}
}
func TestStatsCount_SetBitmapAttrs(t *testing.T) {
func TestStatsCount_SetColumnAttrs(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
@ -162,8 +162,8 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) {
frame.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "SetBitmapAttrs" {
t.Errorf("Expected SetBitmapAttrs, Results %s", name)
if name != "SetRowAttrs" {
t.Errorf("Expected SetRowAttrs, Results %s", name)
}
called = true
},

8
test/fragment.go
View file

@ -85,7 +85,7 @@ func (f *Fragment) Reopen() error {
return nil
}
// MustSetBits sets bits on a row. Panic on error.
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
@ -95,8 +95,8 @@ func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
}
}
// MustClearBits clears bits on a row. Panic on error.
func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) {
// MustClearColumns clears columns on a row. Panic on error.
func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.ClearBit(rowID, columnID); err != nil {
panic(err)
@ -126,7 +126,7 @@ func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) {
s.attrs[id] = m
}
// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk.
// GenerateImportFill generates a set of row/col pairs that evenly fill a fragment chunk.
func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
ipct := int(pct * 100)
for i := 0; i < SliceWidth*rowN; i++ {

13
view.go
View file

@ -30,14 +30,13 @@ import (
// View layout modes.
const (
ViewStandard = "standard"
ViewInverse = "inverse"
ViewFieldPrefix = "field_"
)
// IsValidView returns true if name is valid.
func IsValidView(name string) bool {
return name == ViewStandard || name == ViewInverse
return name == ViewStandard
}
// View represents a container for frame data.
@ -252,9 +251,8 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
// Send the create slice message to all nodes.
err := v.broadcaster.SendAsync(
&internal.CreateSliceMessage{
Index: v.index,
Slice: slice,
IsInverse: IsInverseView(v.name),
Index: v.index,
Slice: slice,
})
if err != nil {
return nil, errors.Wrap(err, "sending message")
@ -428,11 +426,6 @@ func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint6
return r, nil
}
// IsInverseView returns true if the view is used for storing an inverted representation.
func IsInverseView(name string) bool {
return strings.HasPrefix(name, ViewInverse)
}
// ViewInfo represents schema information for a view.
type ViewInfo struct {
Name string `json:"name"`

4
view_test.go
View file

@ -72,7 +72,7 @@ func (v *View) Reopen() error {
return v.Open()
}
// MustSetBits sets bits on a row. Panic on error.
// MustSetBits sets columns on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
@ -82,7 +82,7 @@ func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) {
}
}
// MustClearBits clears bits on a row. Panic on error.
// MustClearColumns clears columns on a row. Panic on error.
func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := v.ClearBit(rowID, columnID); err != nil {

3
webui/assets/main.js
View file

@ -581,14 +581,11 @@ function parse_query(query, indexname) {
function parse_options(option_str) {
var int_keys = ["cacheSize"];
var bool_keys = ["inverseEnabled"];
var options = {};
for (var i = 0; i < option_str.length; i++) {
var parts = option_str[i].split('=');
if (int_keys.indexOf(parts[0]) !== -1 ){
options[parts[0]] = Number(parts[1])
} else if (bool_keys.indexOf(parts[0]) !== -1){
options[parts[0]] = (parts[1] == "true")
} else {
options[parts[0]] = parts[1]
}