diff --git a/.travis.yml b/.travis.yml index 6d5674662..fbe1e0c9b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,10 +16,6 @@ install: - make install-dep install-statik vendor generate-statik script: - make test -# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009 -after_success: - - go get github.com/mattn/goveralls - - $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go" before_deploy: - pip install awscli --user `whoami` deploy: @@ -33,6 +29,7 @@ deploy: matrix: allow_failures: - go: master + fast_finish: true notifications: slack: secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI=" diff --git a/api.go b/api.go index e053e11fa..c8b4b6666 100644 --- a/api.go +++ b/api.go @@ -99,9 +99,9 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er return resp, errors.Wrap(err, "parsing") } execOpts := &ExecOptions{ - Remote: req.Remote, - ExcludeAttrs: req.ExcludeAttrs, - ExcludeBits: req.ExcludeBits, + Remote: req.Remote, + ExcludeRowAttrs: req.ExcludeRowAttrs, + ExcludeColumns: req.ExcludeColumns, } results, err := api.Executor.Execute(ctx, req.Index, q, req.Slices, execOpts) if err != nil { @@ -110,7 +110,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er resp.Results = results // Fill column attributes if requested. - if req.ColumnAttrs && !req.ExcludeBits { + if req.ColumnAttrs && !req.ExcludeColumns { // Consolidate all column ids across all calls. var columnIDs []uint64 for _, result := range results { @@ -118,7 +118,7 @@ func (api *API) Query(ctx context.Context, req *QueryRequest) (QueryResponse, er if !ok { continue } - columnIDs = uint64Slice(columnIDs).merge(bm.Bits()) + columnIDs = uint64Slice(columnIDs).merge(bm.Columns()) } // Retrieve column attributes across all calls. @@ -305,7 +305,7 @@ func (api *API) ExportCSV(ctx context.Context, indexName string, frameName strin // Wrap writer with a CSV writer. cw := csv.NewWriter(w) - // Iterate over each bit. + // Iterate over each column. if err := f.ForEachBit(func(rowID, columnID uint64) error { return cw.Write([]string{ strconv.FormatUint(rowID, 10), @@ -431,79 +431,6 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName return blocks, nil } -// RestoreFrame reads all the data that this host should have for a given frame -// from replicas in the cluster and restores that data to it. -func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error { - if err := api.validate(apiRestoreFrame); err != nil { - return errors.Wrap(err, "validating api method") - } - - // Create a client for the remote cluster. - client := NewInternalHTTPClientFromURI(host, api.RemoteClient) - - // Determine the maximum number of slices. - maxSlices, err := client.MaxSliceByIndex(ctx) - if err != nil { - return errors.Wrap(err, "getting max slice") - } - - // Retrieve frame. - f := api.Holder.Frame(indexName, frameName) - if f == nil { - return ErrFrameNotFound - } - - // Retrieve list of all views. - views, err := client.FrameViews(ctx, indexName, frameName) - if err != nil { - return errors.Wrap(err, "getting views") - } - - // Loop over each slice and import it if this node owns it. - for slice := uint64(0); slice <= maxSlices[indexName]; slice++ { - // Ignore this slice if we don't own it. - if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) { - continue - } - - // Loop over view names. - for _, view := range views { - // Create view. - v, err := f.CreateViewIfNotExists(view) - if err != nil { - return errors.Wrap(err, "creating view") - } - - // Otherwise retrieve the local fragment. - frag, err := v.CreateFragmentIfNotExists(slice) - if err != nil { - return errors.Wrap(err, "creating fragment") - } - - // Stream backup from remote node. - rd, err := client.BackupSlice(ctx, indexName, frameName, view, slice) - if err != nil { - return errors.Wrap(err, "getting backup") - } else if rd == nil { - continue // slice doesn't exist - } - - // Restore to local frame and always close reader. - if err := func() error { - defer rd.Close() - if _, err := frag.ReadFrom(rd); err != nil { - return errors.Wrap(err, "reading fragment") - } - return nil - }(); err != nil { - return err - } - } - } - - return nil -} - // Hosts returns a list of the hosts in the cluster including their ID, // URL, and which is the coordinator. func (api *API) Hosts(ctx context.Context) []*Node { @@ -784,7 +711,7 @@ func (api *API) Import(ctx context.Context, req internal.ImportRequest) error { // Import into fragment. err = frame.Import(req.RowIDs, req.ColumnIDs, timestamps) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, bits=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, columns=%d, err=%s", req.Index, req.Frame, req.Slice, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -803,7 +730,7 @@ func (api *API) ImportValue(ctx context.Context, req internal.ImportValueRequest // Import into fragment. err = frame.ImportValue(req.Field, req.ColumnIDs, req.Values) if err != nil { - api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, bits=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) + api.Logger.Printf("import error: index=%s, frame=%s, slice=%d, field=%s, columns=%d, err=%s", req.Index, req.Frame, req.Slice, req.Field, len(req.ColumnIDs), err) } return errors.Wrap(err, "importing") } @@ -813,12 +740,6 @@ func (api *API) MaxSlices(ctx context.Context) map[string]uint64 { return api.Holder.MaxSlices() } -// MaxInverseSlices returns the maximum inverse slice number for each index in a -// map. -func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 { - return api.Holder.MaxInverseSlices() -} - // StatsWithTags returns an instance of whatever implementation of StatsClient // pilosa is using with the given tags. func (api *API) StatsWithTags(tags []string) StatsClient { @@ -968,7 +889,6 @@ const ( //apiLocalID // not implemented //apiLongQueryTime // not implemented apiMarshalFragment - //apiMaxInverseSlices // not implemented //apiMaxSlices // not implemented apiQuery apiRecalculateCaches diff --git a/cache.go b/cache.go index 5305cfef8..06046220a 100644 --- a/cache.go +++ b/cache.go @@ -168,7 +168,7 @@ func NewRankCache(maxEntries uint32) *RankCache { func (c *RankCache) Add(id uint64, n uint64) { c.mu.Lock() defer c.mu.Unlock() - // Ignore if the bit count is below the threshold. + // Ignore if the column count is below the threshold. if n < c.thresholdValue { return } @@ -469,7 +469,7 @@ type BitmapCache interface { // SimpleCache implements BitmapCache // it is meant to be a short-lived cache for cases where writes are continuing to access -// the same bit within a short time frame (i.e. good for write-heavy loads) +// the same row within a short time frame (i.e. good for write-heavy loads) // A read-heavy use case would cause the cache to get bigger, potentially causing the // node to run out of memory. type SimpleCache struct { diff --git a/client.go b/client.go index 0575b838b..a5b630e10 100644 --- a/client.go +++ b/client.go @@ -15,20 +15,17 @@ package pilosa import ( - "archive/tar" "bytes" "context" "encoding/json" "fmt" "io" "io/ioutil" - "log" "math/rand" "net/http" "net/url" "sort" "strconv" - "time" "crypto/tls" @@ -77,16 +74,11 @@ func (c *InternalHTTPClient) Host() *URI { return c.defaultURI } // MaxSliceByIndex returns the number of slices on a server by index. func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx, false) -} - -// MaxInverseSliceByIndex returns the number of inverse slices on a server by index. -func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) { - return c.maxSliceByIndex(ctx, true) + return c.maxSliceByIndex(ctx) } // maxSliceByIndex returns the number of slices on a server by index. -func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) { +func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) { // Execute request against the host. u := uriPathToURL(c.defaultURI, "/slices/max") @@ -112,9 +104,6 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) return nil, fmt.Errorf("json decode: %s", err) } - if inverse { - return rsp.Inverse, nil - } return rsp.Standard, nil } @@ -308,15 +297,15 @@ func (c *InternalHTTPClient) Import(ctx context.Context, index, frame string, sl return nil } -// ImportK bulk imports bits to a host. -func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, bits []Bit) error { +// ImportK bulk imports bits specified by string keys to a host. +func (c *InternalHTTPClient) ImportK(ctx context.Context, index, frame string, columns []Bit) error { if index == "" { return ErrIndexRequired } else if frame == "" { return ErrFrameRequired } - buf, err := marshalImportPayloadK(index, frame, bits) + buf, err := marshalImportPayloadK(index, frame, columns) if err != nil { return fmt.Errorf("Error Creating Payload: %s", err) } @@ -356,7 +345,7 @@ func marshalImportPayload(index, frame string, slice uint64, bits []Bit) ([]byte columnIDs := Bits(bits).ColumnIDs() timestamps := Bits(bits).Timestamps() - // Marshal bits to protobufs. + // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, Frame: frame, @@ -378,7 +367,7 @@ func marshalImportPayloadK(index, frame string, bits []Bit) ([]byte, error) { columnKeys := Bits(bits).ColumnKeys() timestamps := Bits(bits).Timestamps() - // Marshal bits to protobufs. + // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportRequest{ Index: index, Frame: frame, @@ -465,7 +454,7 @@ func marshalImportValuePayload(index, frame, field string, slice uint64, vals [] columnIDs := FieldValues(vals).ColumnIDs() values := FieldValues(vals).Values() - // Marshal bits to protobufs. + // Marshal data to protobuf. buf, err := proto.Marshal(&internal.ImportValueRequest{ Index: index, Frame: frame, @@ -524,7 +513,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s return ErrIndexRequired } else if frame == "" { return ErrFrameRequired - } else if !(view == ViewStandard || view == ViewInverse) { + } else if view != ViewStandard { return ErrInvalidView } @@ -589,109 +578,6 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde return nil } -// BackupTo backs up an entire frame from a cluster to w. -func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - // Create tar writer around writer. - tw := tar.NewWriter(w) - - // Find the maximum number of slices. - var maxSlices map[string]uint64 - var err error - if view == ViewStandard { - maxSlices, err = c.MaxSliceByIndex(ctx) - } else if view == ViewInverse { - maxSlices, err = c.MaxInverseSliceByIndex(ctx) - } else { - return ErrInvalidView - } - - if err != nil { - return fmt.Errorf("slice n: %s", err) - } - - // Backup every slice to the tar file. - for i := uint64(0); i <= maxSlices[index]; i++ { - if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil { - return errors.Wrap(err, "backing up slice") - } - } - - // Close tar file. - if err := tw.Close(); err != nil { - return errors.Wrap(err, "closing") - } - - return nil -} - -// backupSliceTo backs up a single slice to tw. -func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error { - // Return error if unable to backup from any slice. - r, err := c.BackupSlice(ctx, index, frame, view, slice) - if err != nil { - return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err) - } else if r == nil { - return nil - } - defer r.Close() - - // Read entire buffer to determine file size. - data, err := ioutil.ReadAll(r) - if err != nil { - return errors.Wrap(err, "reading") - } else if err := r.Close(); err != nil { - return errors.Wrap(err, "closing") - } - - // Write slice file header. - if err := tw.WriteHeader(&tar.Header{ - Name: strconv.FormatUint(slice, 10), - Mode: 0666, - Size: int64(len(data)), - ModTime: time.Now(), - }); err != nil { - return errors.Wrap(err, "writing header") - } - - // Write buffer to file. - if _, err := tw.Write(data); err != nil { - return errors.Wrap(err, "writing buffer") - } - - return nil -} - -// BackupSlice retrieves a streaming backup from a single slice. -// This function tries slice owners until one succeeds. -func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) { - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return nil, fmt.Errorf("slice nodes: %s", err) - } - - // Try to backup slice from each one until successful. - for _, i := range rand.Perm(len(nodes)) { - r, err := c.backupSliceNode(ctx, index, frame, view, slice, nodes[i]) - if err == nil { - return r, nil // successfully attached - } else if err == ErrFragmentNotFound { - return nil, nil // slice doesn't exist - } else if err != nil { - log.Println(err) - continue - } - } - - return nil, fmt.Errorf("unable to connect to any owner") -} - func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) { node := &Node{ URI: uri, @@ -734,86 +620,6 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame, return resp.Body, nil } -// RestoreFrom restores a frame from a backup file to an entire cluster. -func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error { - if index == "" { - return ErrIndexRequired - } else if frame == "" { - return ErrFrameRequired - } - - // Create tar reader around input. - tr := tar.NewReader(r) - - // Process each file. - for { - hdr, err := tr.Next() - if err == io.EOF { - return nil - } else if err != nil { - return errors.Wrap(err, "opening") - } - - // Parse slice from entry name. - slice, err := strconv.ParseUint(hdr.Name, 10, 64) - if err != nil { - return fmt.Errorf("invalid backup entry: %s", hdr.Name) - } - - // Read file into buffer. - var buf bytes.Buffer - if _, err := io.CopyN(&buf, tr, hdr.Size); err != nil { - return errors.Wrap(err, "copying") - } - - // Restore file to all nodes that own it. - if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil { - return errors.Wrap(err, "restoring") - } - } -} - -// restoreSliceFrom restores a single slice to all owning nodes. -func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error { - // Retrieve a list of nodes that own the slice. - nodes, err := c.FragmentNodes(ctx, index, slice) - if err != nil { - return fmt.Errorf("slice nodes: %s", err) - } - - // Restore slice to each owner. - for _, node := range nodes { - u := nodePathToURL(node, "/fragment/data") - u.RawQuery = url.Values{ - "index": {index}, - "frame": {frame}, - "view": {view}, - "slice": {strconv.FormatUint(slice, 10)}, - }.Encode() - - // Build request. - req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf)) - if err != nil { - return errors.Wrap(err, "creating request") - } - req.Header.Set("Content-Type", "application/octet-stream") - req.Header.Set("User-Agent", "pilosa/"+Version) - - resp, err := c.HTTPClient.Do(req.WithContext(ctx)) - if err != nil { - return errors.Wrap(err, "executing request") - } - resp.Body.Close() - - // Return error if response not OK. - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode) - } - } - - return nil -} - // CreateFrame creates a new frame on the server. func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error { if index == "" { @@ -1140,7 +946,8 @@ func (c *InternalHTTPClient) SendMessage(ctx context.Context, uri *URI, pb proto return nil } -// Bit represents the location of a single bit. +// Bit represents the intersection of a row and a column. It can be specifed by +// integer ids or string keys. type Bit struct { RowID uint64 ColumnID uint64 @@ -1149,7 +956,7 @@ type Bit struct { Timestamp int64 } -// Bits represents a slice of bits. +// Bits is a slice of Bit. type Bits []Bit func (p Bits) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -1277,7 +1084,7 @@ func (p FieldValues) GroupBySlice() map[uint64][]FieldValue { return m } -// BitsByPos represents a slice of bits sorted by internal position. +// BitsByPos is a slice of bits sorted row then column. type BitsByPos []Bit func (p BitsByPos) Swap(i, j int) { p[i], p[j] = p[j], p[i] } @@ -1314,7 +1121,6 @@ func nodePathToURL(node *Node, path string) url.URL { // I don't want to let it go unquestioned. type InternalClient interface { MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) - MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) Schema(ctx context.Context) ([]*IndexInfo, error) CreateIndex(ctx context.Context, index string, opt IndexOptions) error FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error) @@ -1326,9 +1132,6 @@ type InternalClient interface { EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error - BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error - BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) - RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error RestoreFrame(ctx context.Context, host, index, frame string) error FrameViews(ctx context.Context, index, frame string) ([]string, error) diff --git a/client_test.go b/client_test.go index e2b806412..e4287726b 100644 --- a/client_test.go +++ b/client_test.go @@ -15,7 +15,6 @@ package pilosa_test import ( - "bytes" "context" "fmt" "net/http" @@ -232,65 +231,11 @@ func TestClient_Import(t *testing.T) { } // Verify data. - if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{1, 5}) { - t.Fatalf("unexpected bits: %+v", a) + if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{1, 5}) { + t.Fatalf("unexpected columns: %+v", a) } - if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{6}) { - t.Fatalf("unexpected bits: %+v", a) - } -} - -// Ensure client can bulk import data to an inverse frame. -func TestClient_ImportInverseEnabled(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frameOpts := pilosa.FrameOptions{ - InverseEnabled: true, - } - frame, err := idx.CreateFrameIfNotExists("f", frameOpts) - if err != nil { - panic(err) - } - v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse) - if err != nil { - panic(err) - } - f, err := v.CreateFragmentIfNotExists(0) - if err != nil { - panic(err) - } - - // Load bitmap into cache to ensure cache gets updated. - f.Row(0) - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - // Send import request. - c := test.MustNewClient(s.Host(), defaultClient) - if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{ - {RowID: 0, ColumnID: 1}, - {RowID: 0, ColumnID: 5}, - {RowID: 200, ColumnID: 5}, - {RowID: 200, ColumnID: 6}, - }); err != nil { - t.Fatal(err) - } - - // Verify data. - if a := f.Row(1).Bits(); !reflect.DeepEqual(a, []uint64{0}) { - t.Fatalf("unexpected bits: %+v", a) - } - if a := f.Row(5).Bits(); !reflect.DeepEqual(a, []uint64{0, 200}) { - t.Fatalf("unexpected bits: %+v", a) - } - if a := f.Row(6).Bits(); !reflect.DeepEqual(a, []uint64{200}) { - t.Fatalf("unexpected bits: %+v", a) + if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{6}) { + t.Fatalf("unexpected columns: %+v", a) } } @@ -370,133 +315,6 @@ func TestClient_ImportValue(t *testing.T) { } } -// Ensure client backup and restore a frame. -func TestClient_BackupRestore(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1) - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000) - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - c := test.MustNewClient(s.Host(), defaultClient) - - // Backup from frame. - var buf bytes.Buffer - if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewStandard); err != nil { - t.Fatal(err) - } - - // Restore to a different frame. - if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil { - t.Fatal(err) - } - - // Verify data. - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { - t.Fatalf("unexpected bits(0): %+v", a) - } - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) { - t.Fatalf("unexpected bits(0): %+v", a) - } - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) { - t.Fatalf("unexpected bits(0): %+v", a) - } - if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Bits(); !reflect.DeepEqual(a, []uint64{20000}) { - t.Fatalf("unexpected bits: %+v", a) - } -} - -// Ensure client backup and restore a frame with inverse view. -func TestClient_BackupInverseView(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - frameOpts := pilosa.FrameOptions{ - InverseEnabled: true, - } - frame, err := idx.CreateFrameIfNotExists("f", frameOpts) - if err != nil { - panic(err) - } - v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse) - if err != nil { - panic(err) - } - f, err := v.CreateFragmentIfNotExists(0) - if err != nil { - panic(err) - } - - f.SetBit(100, 1) - f.SetBit(100, 2) - f.SetBit(100, 3) - f.SetBit(100, SliceWidth-1) - - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - c := test.MustNewClient(s.Host(), defaultClient) - - // Backup from frame. - var buf bytes.Buffer - if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewInverse); err != nil { - t.Fatal(err) - } - - // Restore to a different frame. - if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{InverseEnabled: true}); err != nil { - t.Fatal(err) - } - if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewInverse); err != nil { - t.Fatal(err) - } - - // Verify data. - if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Bits(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) { - t.Fatalf("unexpected bits(0): %+v", a) - } - -} - -// backup returns error with invalid view -func TestClient_BackupInvalidView(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1) - - s := test.NewServer() - defer s.Close() - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - c := test.MustNewClient(s.Host(), defaultClient) - - // Backup from frame. - var buf bytes.Buffer - err := c.BackupTo(context.Background(), &buf, "i", "f", "invalid_view") - if err != pilosa.ErrInvalidView { - t.Fatal(err) - } -} - // Ensure client can retrieve a list of all checksums for blocks in a fragment. func TestClient_FragmentBlocks(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/cluster.go b/cluster.go index 97afa0eef..d6026274a 100644 --- a/cluster.go +++ b/cluster.go @@ -626,21 +626,14 @@ func (a viewsByFrame) addView(frame, view string) { func (c *Cluster) fragsByHost(idx *Index) fragsByHost { // frameViews is a map of frame to slice of views. frameViews := make(viewsByFrame) - inverseFrameViews := make(viewsByFrame) for _, frame := range idx.Frames() { for _, view := range frame.Views() { - if IsInverseView(view.Name()) { - inverseFrameViews.addView(frame.Name(), view.Name()) - } else { - frameViews.addView(frame.Name(), view.Name()) - } + frameViews.addView(frame.Name(), view.Name()) + } } - - std := c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews) - inv := c.fragCombos(idx.Name(), idx.MaxInverseSlice(), inverseFrameViews) - return std.add(inv) + return c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews) } // fragCombos returns a map (by uri) of lists of fragments for a given index diff --git a/cluster_test.go b/cluster_test.go index b09820ac8..913cd5db3 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -424,8 +424,6 @@ func TestCluster_ResizeStates(t *testing.T) { // Add Field Data to node0. if err := tc.CreateFrame("i", "fields", FrameOptions{ - InverseEnabled: false, - //CacheType: CacheTypeNone, Fields: []*Field{ { Name: "fld0", diff --git a/cmd/backup.go b/cmd/backup.go deleted file mode 100644 index 4bbbf70c2..000000000 --- a/cmd/backup.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "io" - "os" - - "github.com/pilosa/pilosa/ctl" - "github.com/spf13/cobra" -) - -var Backuper *ctl.BackupCommand - -func NewBackupCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Backuper = ctl.NewBackupCommand(os.Stdin, os.Stdout, os.Stderr) - backupCmd := &cobra.Command{ - Use: "backup", - Short: "Backup data from pilosa.", - Long: ` -Backs up the view from across the cluster into a single file. -`, - RunE: func(cmd *cobra.Command, args []string) error { - if err := Backuper.Run(context.Background()); err != nil { - return err - } - return nil - }, - } - flags := backupCmd.Flags() - flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup.") - flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup.") - flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup.") - flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout") - ctl.SetTLSConfig(flags, &Backuper.TLS.CertificatePath, &Backuper.TLS.CertificateKeyPath, &Backuper.TLS.SkipVerify) - - return backupCmd -} - -func init() { - subcommandFns["backup"] = NewBackupCmd -} diff --git a/cmd/backup_test.go b/cmd/backup_test.go deleted file mode 100644 index 58d9bf21c..000000000 --- a/cmd/backup_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/cmd" -) - -func TestBackupHelp(t *testing.T) { - output, err := ExecNewRootCommand(t, "backup", "--help") - if !strings.Contains(output, "Usage:") || - !strings.Contains(output, "Flags:") || - !strings.Contains(output, "pilosa backup") || err != nil { - t.Fatalf("Command 'backup --help' not working, err: '%v', output: '%s'", err, output) - } -} - -func TestBackupConfig(t *testing.T) { - tests := []commandTest{ - { - args: []string{"backup", "--output-file", "/somefile"}, - env: map[string]string{"PILOSA_HOST": "localhost:12345"}, - cfgFileContent: ` -index = "myindex" -frame = "f1" -`, - validation: func() error { - v := validator{} - v.Check(cmd.Backuper.Host, "localhost:12345") - v.Check(cmd.Backuper.Index, "myindex") - v.Check(cmd.Backuper.Frame, "f1") - v.Check(cmd.Backuper.Path, "/somefile") - return v.Error() - }, - }, - } - executeDry(t, tests) -} diff --git a/cmd/export.go b/cmd/export.go index c5907fbea..951c4a0a9 100644 --- a/cmd/export.go +++ b/cmd/export.go @@ -53,7 +53,6 @@ The file does not contain any headers. flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.") flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export") flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export") - flags.StringVarP(&Exporter.View, "view", "v", "standard", "View to export - default standard") flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout") ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify) diff --git a/cmd/export_test.go b/cmd/export_test.go index 2d01cf4d6..2b30116c6 100644 --- a/cmd/export_test.go +++ b/cmd/export_test.go @@ -44,7 +44,6 @@ frame = "f1" v.Check(cmd.Exporter.Host, "localhost:12345") v.Check(cmd.Exporter.Index, "myindex") v.Check(cmd.Exporter.Frame, "f1") - v.Check(cmd.Exporter.View, "standard") v.Check(cmd.Exporter.Path, "/somefile") return v.Error() }, @@ -52,10 +51,3 @@ frame = "f1" } executeDry(t, tests) } - -func TestExportInvalidView(t *testing.T) { - output, err := ExecNewRootCommand(t, "export", "-i", "foo", "-f", "bar", "-v", "test") - if !strings.Contains(err.Error(), "invalid view") { - t.Fatalf("Command 'export' with invalid view should error but: err: '%v', output: '%v'", err, output) - } -} diff --git a/cmd/import.go b/cmd/import.go index fe01738ad..8dd31181c 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -32,7 +32,7 @@ func NewImportCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command importCmd := &cobra.Command{ Use: "import", Short: "Bulk load data into pilosa.", - Long: `Bulk imports one or more CSV files to a host's index and frame. The bits + Long: `Bulk imports one or more CSV files to a host's index and frame. The data of the CSV file are grouped by slice for the most efficient import. The format of the CSV file is: @@ -61,7 +61,6 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.BoolVarP(&Importer.Sort, "sort", "", false, "Enables sorting before import.") flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") flags.Var(&Importer.FrameOptions.TimeQuantum, "frame-time-quantum", "Time quantum for the frame") - flags.BoolVar(&Importer.FrameOptions.RangeEnabled, "frame-range-enabled", false, "DEPRECATED - any frame can have fields. This option will be removed.") flags.StringVar(&Importer.FrameOptions.CacheType, "frame-cache-type", pilosa.CacheTypeRanked, "Cache type for the frame; valid values: none, lru, ranked") flags.Uint32Var(&Importer.FrameOptions.CacheSize, "frame-cache-size", 50000, "Cache size for the frame") ctl.SetTLSConfig(flags, &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.SkipVerify) diff --git a/cmd/restore.go b/cmd/restore.go deleted file mode 100644 index e5169be12..000000000 --- a/cmd/restore.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "io" - "os" - - "github.com/spf13/cobra" - - "github.com/pilosa/pilosa/ctl" -) - -var Restorer *ctl.RestoreCommand - -func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command { - Restorer = ctl.NewRestoreCommand(os.Stdin, os.Stdout, os.Stderr) - - restoreCmd := &cobra.Command{ - Use: "restore", - Short: "Restore data to pilosa from a backup file.", - Long: ` -Restores a view to the cluster from a backup file. -`, - RunE: func(cmd *cobra.Command, args []string) error { - if err := Restorer.Run(context.Background()); err != nil { - return err - } - return nil - }, - } - flags := restoreCmd.Flags() - flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.") - flags.StringVarP(&Restorer.Index, "index", "i", "", "Pilosa index to restore into.") - flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.") - flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.") - flags.StringVarP(&Restorer.Path, "input-file", "d", "", "File to restore data from.") - ctl.SetTLSConfig(flags, &Restorer.TLS.CertificatePath, &Restorer.TLS.CertificateKeyPath, &Restorer.TLS.SkipVerify) - - return restoreCmd -} - -func init() { - subcommandFns["restore"] = NewRestoreCommand -} diff --git a/cmd/restore_test.go b/cmd/restore_test.go deleted file mode 100644 index cdad8ca92..000000000 --- a/cmd/restore_test.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd_test - -import ( - "strings" - "testing" - - "github.com/pilosa/pilosa/cmd" -) - -func TestRestoreHelp(t *testing.T) { - output, err := ExecNewRootCommand(t, "restore", "--help") - if !strings.Contains(output, "Usage:") || - !strings.Contains(output, "Flags:") || - !strings.Contains(output, "pilosa restore") || err != nil { - t.Fatalf("Command 'restore --help' not working, err: '%v', output: '%s'", err, output) - } -} - -func TestRestoreConfig(t *testing.T) { - tests := []commandTest{ - { - args: []string{"restore", "--input-file", "/somefile"}, - env: map[string]string{"PILOSA_HOST": "localhost:12345"}, - cfgFileContent: ` -index = "myindex" -frame = "f1" -`, - validation: func() error { - v := validator{} - v.Check(cmd.Restorer.Host, "localhost:12345") - v.Check(cmd.Restorer.Index, "myindex") - v.Check(cmd.Restorer.Frame, "f1") - v.Check(cmd.Restorer.Path, "/somefile") - return v.Error() - }, - }, - } - executeDry(t, tests) -} diff --git a/ctl/backup.go b/ctl/backup.go deleted file mode 100644 index 54a760695..000000000 --- a/ctl/backup.go +++ /dev/null @@ -1,94 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "context" - "io" - "os" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/server" - "github.com/pkg/errors" -) - -// BackupCommand represents a command for backing up a view. -type BackupCommand struct { - // Destination host and port. - Host string - - // Name of the index, frame, view to backup. - Index string - Frame string - View string - - // Output file to write to. - Path string - - // Standard input/output - *pilosa.CmdIO - - TLS server.TLSConfig -} - -// NewBackupCommand returns a new instance of BackupCommand. -func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand { - return &BackupCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - } -} - -// Run executes the backup. -func (cmd *BackupCommand) Run(ctx context.Context) error { - // Validate arguments. - if cmd.Path == "" { - return errors.New("output file required") - } - - // Create a client to the server. - client, err := CommandClient(cmd) - if err != nil { - return errors.Wrap(err, "creating client") - } - - // Open output file. - f, err := os.Create(cmd.Path) - if err != nil { - return errors.Wrap(err, "creating file") - } - defer f.Close() - - // Begin streaming backup. - if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { - return errors.Wrap(err, "backing up") - } - - // Sync & close file to ensure durability. - if err := f.Sync(); err != nil { - return errors.Wrap(err, "syncing") - } else if err = f.Close(); err != nil { - return errors.Wrap(err, "closing file") - } - - return nil -} - -func (cmd *BackupCommand) TLSHost() string { - return cmd.Host -} - -func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig { - return cmd.TLS -} diff --git a/ctl/backup_test.go b/ctl/backup_test.go deleted file mode 100644 index a073bc8a8..000000000 --- a/ctl/backup_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "bytes" - "context" - "io/ioutil" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" -) - -func TestBackupCommand_FileRequired(t *testing.T) { - buf := bytes.Buffer{} - stdin, stdout, stderr := GetIO(buf) - - cm := NewBackupCommand(stdin, stdout, stderr) - err := cm.Run(context.Background()) - if err.Error() != "output file required" { - t.Fatalf("expect error: output file required, actual: %s", err) - } - -} - -func TestBackupCommand_Run(t *testing.T) { - - buf := bytes.Buffer{} - stdin, stdout, stderr := GetIO(buf) - - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - cm := NewBackupCommand(stdin, stdout, stderr) - file, err := ioutil.TempFile("", "import.csv") - - cm.Index = "i" - cm.Host = s.Host() - cm.Frame = "f" - cm.View = pilosa.ViewStandard - cm.Path = file.Name() - err = cm.Run(context.Background()) - if err != nil { - t.Fatalf("Command not working, error: '%s'", err) - } -} diff --git a/ctl/bench_test.go b/ctl/bench_test.go index 4790ccb44..766ec8a1a 100644 --- a/ctl/bench_test.go +++ b/ctl/bench_test.go @@ -15,6 +15,7 @@ package ctl import ( + "bufio" "bytes" "context" "fmt" @@ -86,3 +87,12 @@ func TestBenchCommand_Run(t *testing.T) { fmt.Println(buf.String()) } } + +// declare stdin, stdout, stderr +func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { + rder := []byte{} + stdin := bytes.NewReader(rder) + stdout := bufio.NewWriter(&buf) + stderr := bufio.NewWriter(&buf) + return stdin, stdout, stderr +} diff --git a/ctl/export.go b/ctl/export.go index 546d1ae1b..2c40b3bc4 100644 --- a/ctl/export.go +++ b/ctl/export.go @@ -33,7 +33,7 @@ type ExportCommand struct { // Name of the index & frame to export from. Index string Frame string - View string + // Filename to export to. Path string @@ -59,8 +59,6 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { return pilosa.ErrIndexRequired } else if cmd.Frame == "" { return pilosa.ErrFrameRequired - } else if !(cmd.View == pilosa.ViewStandard || cmd.View == pilosa.ViewInverse) { - return pilosa.ErrInvalidView } // Use output file, if specified. @@ -83,13 +81,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { } // Determine slice count. - var maxSlices map[string]uint64 - if cmd.View == pilosa.ViewStandard { - maxSlices, err = client.MaxSliceByIndex(ctx) - } else if cmd.View == pilosa.ViewInverse { - maxSlices, err = client.MaxInverseSliceByIndex(ctx) - } - + maxSlices, err := client.MaxSliceByIndex(ctx) if err != nil { return errors.Wrap(err, "getting slice count") } @@ -97,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error { // Export each slice. for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ { logger.Printf("exporting slice: %d", slice) - if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil { + if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, pilosa.ViewStandard, slice, w); err != nil { return errors.Wrap(err, "exporting") } } diff --git a/ctl/export_test.go b/ctl/export_test.go index 48a90a065..469ca7d75 100644 --- a/ctl/export_test.go +++ b/ctl/export_test.go @@ -41,13 +41,6 @@ func TestExportCommand_Validation(t *testing.T) { if err != pilosa.ErrFrameRequired { t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err) } - - cm.Frame = "f" - cm.View = "test" - err = cm.Run(context.Background()) - if err != pilosa.ErrInvalidView { - t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrInvalidView, err) - } } func TestExportCommand_Run(t *testing.T) { @@ -70,7 +63,6 @@ func TestExportCommand_Run(t *testing.T) { cm.Index = "i" cm.Frame = "f" - cm.View = pilosa.ViewStandard if err := cm.Run(context.Background()); err != nil { t.Fatalf("Export Run doesn't work: %s", err) } diff --git a/ctl/import.go b/ctl/import.go index 004e47e96..a9d70fd7c 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -107,7 +107,6 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { // Import each path and import by slice. for _, path := range cmd.Paths { - // Parse path into bits. logger.Printf("parsing: %s", path) if err := cmd.importPath(ctx, path); err != nil { return err @@ -236,13 +235,13 @@ func (cmd *ImportCommand) importBits(ctx context.Context, bits []pilosa.Bit) err bitsBySlice := pilosa.Bits(bits).GroupBySlice() // Parse path into bits. - for slice, bits := range bitsBySlice { + for slice, chunk := range bitsBySlice { if cmd.Sort { - sort.Sort(pilosa.BitsByPos(bits)) + sort.Sort(pilosa.BitsByPos(chunk)) } - logger.Printf("importing slice: %d, n=%d", slice, len(bits)) - if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, bits); err != nil { + logger.Printf("importing slice: %d, n=%d", slice, len(chunk)) + if err := cmd.Client.Import(ctx, cmd.Index, cmd.Frame, slice, chunk); err != nil { return errors.Wrap(err, "importing") } } diff --git a/ctl/restore.go b/ctl/restore.go deleted file mode 100644 index ef863096e..000000000 --- a/ctl/restore.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "context" - "io" - "os" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/server" - "github.com/pkg/errors" -) - -// RestoreCommand represents a command for restoring a frame from a backup. -type RestoreCommand struct { - // Destination host and port. - Host string - - // Name of the index & frame to backup. - Index string - Frame string - View string - - // Import file to read from. - Path string - - // Standard input/output - *pilosa.CmdIO - - TLS server.TLSConfig -} - -// NewRestoreCommand returns a new instance of RestoreCommand. -func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand { - return &RestoreCommand{ - CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr), - } -} - -// Run executes the restore command. -func (cmd *RestoreCommand) Run(ctx context.Context) error { - // Validate arguments. - if cmd.Path == "" { - return errors.New("backup file required") - } - - // Create a client to the server. - client, err := CommandClient(cmd) - if err != nil { - return errors.Wrap(err, "creating client") - } - - // Open backup file. - f, err := os.Open(cmd.Path) - if err != nil { - return errors.Wrap(err, "opening file") - } - defer f.Close() - - // Restore backup file to the cluster. - if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil { - return errors.Wrap(err, "restoring") - } - - return nil -} - -func (cmd *RestoreCommand) TLSHost() string { - return cmd.Host -} - -func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig { - return cmd.TLS -} diff --git a/ctl/restore_test.go b/ctl/restore_test.go deleted file mode 100644 index 896db57c5..000000000 --- a/ctl/restore_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ctl - -import ( - "bufio" - "bytes" - "context" - "io" - "io/ioutil" - "testing" - - "github.com/pilosa/pilosa" - "github.com/pilosa/pilosa/test" -) - -func TestRestoreCommand_FileRequired(t *testing.T) { - cm := RestoreCommand{} - ctx := context.Background() - err := cm.Run(ctx) - if err.Error() != "backup file required" { - t.Fatalf("expect error: output file required, actual: '%s'", err) - } -} - -func TestRestoreCommand_Run(t *testing.T) { - file, err := ioutil.TempFile("", "restore.csv") - if err != nil { - t.Fatal(err) - } - buf := bytes.Buffer{} - stdin, stdout, stderr := GetIO(buf) - - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - defer s.Close() - - s.Handler.API.Cluster = test.NewCluster(1) - s.Handler.API.Cluster.Nodes[0].URI = s.HostURI() - s.Handler.API.Holder = hldr.Holder - - cm := NewRestoreCommand(stdin, stdout, stderr) - cm.Path = file.Name() - cm.Index = "i" - cm.Frame = "f" - cm.View = pilosa.ViewStandard - cm.Host = s.Host() - cm.Run(context.Background()) - if err != nil { - t.Fatalf("Backup Run doesn't work: %s", err) - } -} - -// declare stdin, stdout, stderr -func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) { - rder := []byte{} - stdin := bytes.NewReader(rder) - stdout := bufio.NewWriter(&buf) - stderr := bufio.NewWriter(&buf) - return stdin, stdout, stderr -} diff --git a/docs/administration.md b/docs/administration.md index 5a469c85e..d25f8a8d1 100644 --- a/docs/administration.md +++ b/docs/administration.md @@ -65,7 +65,7 @@ pilosa import -i project -f stargazer --field star_count project-stargazer-count ```
-

Note that you must first create a frame with range-encoding enabled and a field. View Create Frame for more details.

+

Note that you must first create a frame and a field. View Create Frame for more details.

#### Exporting diff --git a/docs/api-reference.md b/docs/api-reference.md index 7841dff2e..f94cc5009 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -105,7 +105,6 @@ The request payload is in JSON, and may contain the `options` field. The `option * `timeQuantum` (string): [Time Quantum](../data-model/#time-quantum) for this frame. * `cacheType` (string): [ranked](../data-model/#ranked) or [LRU](../data-model/#lru) caching on this frame. Default is `lru`. * `cacheSize` (int): Number of rows to keep in the cache. Default 50,000. -* `rangeEnabled` (boolean): DEPRECATED - has no effect, will be removed. All frames support BSI fields. * `fields` (array): List of range-encoded [fields](../data-model/#bsi-range-encoding). Each individual `field` contains the following: diff --git a/docs/installation.md b/docs/installation.md index c4f30ed8d..039ea370d 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -58,7 +57,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: @@ -110,7 +108,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -119,7 +116,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: @@ -177,7 +173,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -186,7 +181,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/) help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: @@ -268,7 +262,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -277,7 +270,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: @@ -335,7 +327,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende pilosa [command] Available Commands: - backup Backup data from pilosa. bench Benchmark operations. check Do a consistency check on a pilosa data file. config Print the current configuration. @@ -344,7 +335,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende help Help about any command import Bulk load data into pilosa. inspect Get stats on a pilosa data file. - restore Restore data to pilosa from a backup file. server Run Pilosa. Flags: diff --git a/executor.go b/executor.go index 2b9f129e4..59cd385a6 100644 --- a/executor.go +++ b/executor.go @@ -80,36 +80,21 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Don't bother calculating slices for query types that don't require it. needsSlices := needsSlices(q.Calls) - // MaxSlice can differ between inverse and standard views, so we need - // to send queries to different slices based on orientation. - var inverseSlices []uint64 - - // If slices are specified, then use that value for slices or - // inverseSlices. If slices aren't specified, then include all of them. - if len(slices) > 0 { - // For inverse queries, the values of `slices` provided to the Execute() method - // on the remote node actually represents inverseSlices. - inverseSlices = slices - } else if needsSlices { + // If slices are specified, then use that value for slices. If slices aren't + // specified, then include all of them. + if len(slices) == 0 && needsSlices { // Round up the number of slices. idx := e.Holder.Index(index) if idx == nil { return nil, ErrIndexNotFound } maxSlice := idx.MaxSlice() - maxInverseSlice := idx.MaxInverseSlice() // Generate a slices of all slices. slices = make([]uint64, maxSlice+1) for i := range slices { slices[i] = uint64(i) } - - // Generate a slices of all inverse slices. - inverseSlices = make([]uint64, maxInverseSlice+1) - for i := range inverseSlices { - inverseSlices[i] = uint64(i) - } } // Optimize handling for bulk attribute insertion. @@ -120,23 +105,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic // Execute each call serially. results := make([]interface{}, 0, len(q.Calls)) for _, call := range q.Calls { - if call.SupportsInverse() && needsSlices { - // Fetch frame & row label based on argument. - frame := call.Args["frame"].(string) - if frame == "" { - frame = DefaultFrame - } - f := e.Holder.Frame(index, frame) - if f == nil { - return nil, ErrFrameNotFound - } - - // If this call is to an inverse frame send to a different list of slices. - if call.IsInverse(rowLabel, columnLabel) { - slices = inverseSlices - } - } - v, err := e.executeCall(ctx, index, call, slices, opt) if err != nil { return nil, err @@ -336,7 +304,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C // If the row label is used then return bitmap attributes. row, _ := other.(*Row) if c.Name == "Bitmap" { - if opt.ExcludeAttrs { + if opt.ExcludeRowAttrs { row.Attrs = map[string]interface{}{} } else { idx := e.Holder.Index(index) @@ -367,7 +335,7 @@ func (e *Executor) executeBitmapCall(ctx context.Context, index string, c *pql.C } } - if opt.ExcludeBits { + if opt.ExcludeColumns { row.segments = []RowSegment{} } @@ -580,7 +548,6 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C // executeTopNSlice executes a TopN call for a single slice. func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) { frame, _ := c.Args["frame"].(string) - inverse, _ := c.Args["inverse"].(bool) n, _, err := c.UintArg("n") if err != nil { return nil, fmt.Errorf("executeTopNSlice: %v", err) @@ -619,9 +586,6 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca // Determine view. view := ViewStandard - if inverse { - view = ViewInverse - } f := e.Holder.Fragment(index, frame, view, slice) if f == nil { @@ -685,32 +649,19 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql. return nil, ErrFrameNotFound } - // Return an error if both the row and column label are specified. rowID, rowOK, rowErr := c.UintArg(rowLabel) - columnID, columnOK, columnErr := c.UintArg(columnLabel) - if rowErr != nil || columnErr != nil { - return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr) + if rowErr != nil { + return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr) } - if rowOK && columnOK { - return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel) - } else if !rowOK && !columnOK { - return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", rowLabel, columnLabel) + if !rowOK { + return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel) } - // Determine row or column orientation. - view, id := ViewStandard, rowID - if columnOK { - view, id = ViewInverse, columnID - if !f.InverseEnabled() { - return nil, fmt.Errorf("Bitmap() cannot retrieve columns unless inverse storage enabled") - } - } - - frag := e.Holder.Fragment(index, frame, view, slice) + frag := e.Holder.Fragment(index, frame, ViewStandard, slice) if frag == nil { return NewRow(), nil } - return frag.Row(id), nil + return frag.Row(rowID), nil } // executeIntersectSlice executes a intersect() call for a local slice. @@ -761,26 +712,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C } // Read row & column id. - columnID, columnOK, err := c.UintArg(columnLabel) - if err != nil { - return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err) - } rowID, rowOK, err := c.UintArg(rowLabel) if err != nil { return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err) } - - // Determine view. - var id uint64 - var viewName string - if columnOK && rowOK { - return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, rowLabel) - } else if !columnOK && !rowOK { - return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, rowLabel) - } else if columnOK { - viewName, id = ViewInverse, columnID - } else { - viewName, id = ViewStandard, rowID + if !rowOK { + return nil, fmt.Errorf("Range() must specify %q", rowLabel) } // Parse start time. @@ -811,12 +748,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C // Union bitmaps across all time-based subframes. row := &Row{} - for _, view := range ViewsByTimeRange(viewName, startTime, endTime, q) { + for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) { f := e.Holder.Fragment(index, frame, view, slice) if f == nil { continue } - row = row.Union(f.Row(id)) + row = row.Union(f.Row(rowID)) } f.Stats.Count("range", 1, 1.0) return row, nil @@ -1033,7 +970,6 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call, // executeClearBit executes a ClearBit() call. func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - view, _ := c.Args["view"].(string) frame, ok := c.Args["frame"].(string) if !ok { return false, errors.New("ClearBit() frame required") @@ -1064,31 +1000,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel) } - // Clear bits for each view. - switch view { - case ViewStandard: - return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt) - case ViewInverse: - return e.executeClearBitView(ctx, index, c, f, view, rowID, colID, opt) - case "": - var ret bool - if changed, err := e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt); err != nil { - return ret, err - } else if changed { - ret = true - } - - if f.InverseEnabled() { - if changed, err := e.executeClearBitView(ctx, index, c, f, ViewInverse, rowID, colID, opt); err != nil { - return ret, err - } else if changed { - ret = true - } - } - return ret, nil - default: - return false, fmt.Errorf("invalid view: %s", view) - } + return e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt) } // executeClearBitView executes a ClearBit() call for a single view. @@ -1123,7 +1035,6 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql // executeSetBit executes a SetBit() call. func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) { - view, _ := c.Args["view"].(string) frame, ok := c.Args["frame"].(string) if !ok { return false, errors.New("SetBit() field required: frame") @@ -1164,31 +1075,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, timestamp = &t } - // Set bits for each view. - switch view { - case ViewStandard: - return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt) - case ViewInverse: - return e.executeSetBitView(ctx, index, c, f, view, rowID, colID, timestamp, opt) - case "": - var ret bool - if changed, err := e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil { - return ret, err - } else if changed { - ret = true - } - - if f.InverseEnabled() { - if changed, err := e.executeSetBitView(ctx, index, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil { - return ret, err - } else if changed { - ret = true - } - } - return ret, nil - default: - return false, fmt.Errorf("invalid view: %s", view) - } + return e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt) } // executeSetBitView executes a SetBit() call for a specific view. @@ -1319,7 +1206,7 @@ func (e *Executor) executeSetRowAttrs(ctx context.Context, index string, c *pql. if err := frame.RowAttrStore().SetAttrs(rowID, attrs); err != nil { return err } - frame.Stats.Count("SetBitmapAttrs", 1, 1.0) + frame.Stats.Count("SetRowAttrs", 1, 1.0) // Do not forward call if this is already being forwarded. if opt.Remote { @@ -1404,7 +1291,7 @@ func (e *Executor) executeBulkSetRowAttrs(ctx context.Context, index string, cal if err := frame.RowAttrStore().SetBulkAttrs(frameMap); err != nil { return nil, err } - frame.Stats.Count("SetBitmapAttrs", 1, 1.0) + frame.Stats.Count("SetRowAttrs", 1, 1.0) } // Do not forward call if this is already being forwarded. @@ -1700,9 +1587,9 @@ type mapResponse struct { // ExecOptions represents an execution context for a single Execute() call. type ExecOptions struct { - Remote bool - ExcludeAttrs bool - ExcludeBits bool + Remote bool + ExcludeRowAttrs bool + ExcludeColumns bool } // decodeError returns an error representation of s if s is non-blank. diff --git a/executor_test.go b/executor_test.go index 7948ed2fe..0807228ac 100644 --- a/executor_test.go +++ b/executor_test.go @@ -33,7 +33,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) + f, err := index.CreateFrame("f", pilosa.FrameOptions{}) if err != nil { t.Fatal(err) } @@ -54,26 +54,26 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if bits := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", bits) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } - // Inhibit bits. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeBits: true}); err != nil { + // Inhibit column attributes. + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + t.Fatalf("unexpected columns: %+v", columns) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } - // Inhibit attributes. - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeAttrs: true}); err != nil { + // Inhibit row attributes. + if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeRowAttrs: true}); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{3, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{}) { t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) } @@ -83,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + if _, err := index.CreateFrame("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } @@ -100,14 +100,6 @@ func TestExecutor_Execute_Bitmap(t *testing.T) { if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil { t.Fatal(err) } - - if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil { - t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) { - t.Fatalf("unexpected bits: %+v", bits) - } else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) { - t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs)) - } }) } @@ -124,8 +116,8 @@ func TestExecutor_Execute_Difference(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 3}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -156,8 +148,8 @@ func TestExecutor_Execute_Intersect(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 2}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -186,8 +178,8 @@ func TestExecutor_Execute_Union(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -200,8 +192,8 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -219,8 +211,8 @@ func TestExecutor_Execute_Xor(t *testing.T) { e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Xor(Bitmap(row=10), Bitmap(row=11))`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{0, 2, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -255,7 +247,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatal(err) } else { if !res[0].(bool) { - t.Fatalf("expected bit changed") + t.Fatalf("expected column changed") } } @@ -266,7 +258,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) { t.Fatal(err) } else { if res[0].(bool) { - t.Fatalf("expected bit unchanged") + t.Fatalf("expected column unchanged") } } } @@ -409,12 +401,12 @@ func TestExecutor_Execute_TopN(t *testing.T) { defer hldr.Close() e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - // Set bits for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two slices. if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + } else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil { t.Fatal(err) - } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + } else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if _, err := e.Execute(context.Background(), "i", test.MustParse(` SetBit(frame=f, row=0, col=0) @@ -431,7 +423,6 @@ func TestExecutor_Execute_TopN(t *testing.T) { } hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache() - hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewInverse, 0).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache() hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache() @@ -445,24 +436,13 @@ func TestExecutor_Execute_TopN(t *testing.T) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) - - t.Run("Inverse", func(t *testing.T) { - if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil { - t.Fatal(err) - } else if !reflect.DeepEqual(result[0], []pilosa.Pair{ - {ID: SliceWidth, Count: 3}, - {ID: 0, Count: 2}, - }) { - t.Fatalf("unexpected result: %s", spew.Sdump(result)) - } - }) } func TestExecutor_Execute_TopN_fill(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set bits for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two slices. hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 2) @@ -520,7 +500,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() - // Set bits for rows 0, 10, & 20 across two slices. + // Set columns for rows 0, 10, & 20 across two slices. hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1) hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, SliceWidth) @@ -609,7 +589,6 @@ func TestExecutor_Execute_MinMax(t *testing.T) { } if _, err := idx.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, Fields: []*pilosa.Field{ {Name: "foo", Type: pilosa.FieldTypeInt, Min: -10, Max: 100}, }, @@ -759,13 +738,12 @@ func TestExecutor_Execute_Range(t *testing.T) { // Create frame. if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{ - InverseEnabled: true, - TimeQuantum: pilosa.TimeQuantum("YMDH"), + TimeQuantum: pilosa.TimeQuantum("YMDH"), }); err != nil { t.Fatal(err) } - // Set bits. + // Set columns. if _, err := e.Execute(context.Background(), "i", test.MustParse(` SetBit(frame=f, row=1, col=2, timestamp="1999-12-31T00:00") SetBit(frame=f, row=1, col=3, timestamp="2000-01-01T00:00") @@ -784,19 +762,11 @@ func TestExecutor_Execute_Range(t *testing.T) { t.Run("Standard", func(t *testing.T) { if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(row=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{2, 3, 4, 5, 6, 7}) { + t.Fatalf("unexpected columns: %+v", columns) } }) - t.Run("Inverse", func(t *testing.T) { - e := test.NewExecutor(hldr.Holder, test.NewCluster(1)) - if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil { - t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) { - t.Fatalf("unexpected bits: %+v", bits) - } - }) } // Ensure a Range(field) query can be executed. @@ -854,7 +824,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("EQ", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{50, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -863,28 +833,28 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { // NEQ null if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != null)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo != 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1, SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } // NEQ - if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo != -20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { //t.Fatalf("unexpected result: %s", spew.Sdump(result)) - t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Bits()) + t.Fatalf("unexpected result: %v", result[0].(*pilosa.Row).Columns()) } }) t.Run("LT", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo < 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{SliceWidth + 2}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -892,7 +862,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("LTE", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo <= 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{50, SliceWidth + 2, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -900,7 +870,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("GT", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo > 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{SliceWidth, SliceWidth + 1}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -908,7 +878,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("GTE", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo >= 20)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{50, SliceWidth, SliceWidth + 1, (5 * SliceWidth) + 100}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -916,7 +886,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("BETWEEN", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [1, 1000])`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -925,7 +895,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("FieldNotNull", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=other, foo >< [0, 1000])`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{0}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -933,7 +903,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("BelowMin", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 0)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -941,7 +911,7 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("AboveMax", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=f, foo == 200)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Bits()) { + } else if !reflect.DeepEqual([]uint64{}, result[0].(*pilosa.Row).Columns()) { t.Fatalf("unexpected result: %s", spew.Sdump(result)) } }) @@ -949,16 +919,16 @@ func TestExecutor_Execute_FieldRange(t *testing.T) { t.Run("LTAboveMax", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo < 200)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits())) + } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) } }) t.Run("GTBelowMin", func(t *testing.T) { if result, err := e.Execute(context.Background(), "i", test.MustParse(`Range(frame=edge, foo > -200)`), nil, nil); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Bits()) { - t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Bits())) + } else if !reflect.DeepEqual([]uint64{0, 1}, result[0].(*pilosa.Row).Columns()) { + t.Fatalf("unexpected result: %s", spew.Sdump(result[0].(*pilosa.Row).Columns())) } }) @@ -999,7 +969,7 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { t.Fatalf("unexpected slices: %+v", slices) } - // Set bits in slice 0 & 2. + // Set columns in slice 0 & 2. r := pilosa.NewRow( (0*SliceWidth)+1, (0*SliceWidth)+2, @@ -1018,8 +988,8 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) { e := test.NewExecutor(hldr.Holder, c) if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, nil); err != nil { t.Fatal(err) - } else if bits := res[0].(*pilosa.Row).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 2*SliceWidth + 4}) { + t.Fatalf("unexpected columns: %+v", columns) } } @@ -1058,7 +1028,7 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) { } } -// Ensure a remote query can set bits on multiple nodes. +// Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit(t *testing.T) { c := test.NewCluster(2) c.ReplicaN = 2 @@ -1101,7 +1071,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { t.Fatal(err) } - // Verify that one bit is set on both node's holder. + // Verify that one column is set on both node's holder. if n := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } @@ -1110,7 +1080,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) { } } -// Ensure a remote query can set bits on multiple nodes. +// Ensure a remote query can set columns on multiple nodes. func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { c := test.NewCluster(2) c.ReplicaN = 2 @@ -1155,7 +1125,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) { t.Fatal(err) } - // Verify that one bit is set on both node's holder. + // Verify that one column is set on both node's holder. if n := hldr.MustCreateFragmentIfNotExists("i", "f", "standard_2016", 0).Row(10).Count(); n != 1 { t.Fatalf("unexpected local count: %d", n) } @@ -1247,7 +1217,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{}) - index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}) + index.CreateFrame("f", pilosa.FrameOptions{}) targetAttrs := map[string]interface{}{ "foo": "bar", } diff --git a/fragment.go b/fragment.go index 492524c0d..f4e9906a1 100644 --- a/fragment.go +++ b/fragment.go @@ -393,7 +393,7 @@ func (f *Fragment) setBit(rowID, columnID uint64) (changed bool, err error) { // Determine the position of the bit in the storage. pos, err := f.pos(rowID, columnID) if err != nil { - return false, errors.Wrap(err, "getting bit ops") + return false, errors.Wrap(err, "getting bit pos") } // Write to storage. @@ -585,7 +585,7 @@ func (f *Fragment) importSetFieldValue(columnID uint64, bitDepth uint, value uin // FieldSum returns the sum of a given field as well as the number of columns involved. // A bitmap can be passed in to optionally filter the computed columns. func (f *Fragment) FieldSum(filter *Row, bitDepth uint) (sum, count uint64, err error) { - // Compute count based on the existence bit. + // Compute count based on the existence row. row := f.Row(uint64(bitDepth)) if filter != nil { count = row.IntersectionCount(filter) @@ -629,7 +629,7 @@ func (f *Fragment) FieldMin(filter *Row, bitDepth uint) (min, count uint64, err } for i := bitDepth; i > uint(0); i-- { - ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + ii := i - 1 // allow for uint range: (bitDepth-1) to 0 row := f.Row(uint64(ii)) x := consider.Difference(row) @@ -662,7 +662,7 @@ func (f *Fragment) FieldMax(filter *Row, bitDepth uint) (max, count uint64, err } for i := bitDepth; i > uint(0); i-- { - ii := i - 1 // allow for uint range: (bitdepth-1) to 0 + ii := i - 1 // allow for uint range: (bitDepth-1) to 0 row := f.Row(uint64(ii)) x := row.Intersect(consider) @@ -842,7 +842,7 @@ func (f *Fragment) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax u } // LTE predicateMin - // If bit is zero then remove all set columns not in excluded bitmap. + // If bit is zero then remove all set bits not in excluded bitmap. if bit2 == 0 { b = b.Difference(row.Difference(keep2)) } else { @@ -996,13 +996,13 @@ func (f *Fragment) Top(opt TopOptions) ([]Pair, error) { // If it's too low then don't try finding anymore pairs. threshold := results.Pairs[0].Count - // If the row doesn't have enough bits set before the intersection + // If the row doesn't have enough columns set before the intersection // then we can assume that any remaining rows also have a count too low. if threshold < opt.MinThreshold || cnt < threshold { break } - // Calculate the intersecting bit count and skip if it's below our + // Calculate the intersecting column count and skip if it's below our // last row in our current result set. count := opt.Src.IntersectionCount(f.Row(rowID)) if count < threshold { diff --git a/fragment_test.go b/fragment_test.go index 22c8fe81f..5c5f03b16 100644 --- a/fragment_test.go +++ b/fragment_test.go @@ -176,10 +176,10 @@ func TestFragment_SetFieldValue(t *testing.T) { }) t.Run("QuickCheck", func(t *testing.T) { - if err := quick.Check(func(bitDepth uint, columnN uint64, values []uint64) bool { + if err := quick.Check(func(bitDepth uint, bitN uint64, values []uint64) bool { // Limit bit depth & maximum values. bitDepth = (bitDepth % 62) + 1 - columnN = (columnN % 99) + 1 + bitN = (bitN % 99) + 1 for i := range values { values[i] = values[i] % (1 << bitDepth) @@ -191,7 +191,7 @@ func TestFragment_SetFieldValue(t *testing.T) { // Set values. m := make(map[uint64]int64) for _, value := range values { - columnID := value % columnN + columnID := value % bitN m[columnID] = int64(value) @@ -206,9 +206,9 @@ func TestFragment_SetFieldValue(t *testing.T) { if err != nil { t.Fatal(err) } else if value != int64(v) { - t.Fatalf("value mismatch: column=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v) + t.Fatalf("value mismatch: columnID=%d, bitdepth=%d, value: %d != %d", columnID, bitDepth, value, v) } else if !exists { - t.Fatalf("value should exist: column=%d", columnID) + t.Fatalf("value should exist: columnID=%d", columnID) } } @@ -353,8 +353,8 @@ func TestFragment_FieldRange(t *testing.T) { // Query for equality. if b, err := f.FieldRange(pql.EQ, bitDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) @@ -376,8 +376,8 @@ func TestFragment_FieldRange(t *testing.T) { // Query for inequality. if b, err := f.FieldRange(pql.NEQ, bitDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) @@ -400,32 +400,32 @@ func TestFragment_FieldRange(t *testing.T) { t.Fatal(err) } - // Query for fields less than (ending with set bit). + // Query for fields less than (ending with set column). if b, err := f.FieldRange(pql.LT, bitDepth, 301); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than (ending with unset bit). + // Query for fields less than (ending with unset column). if b, err := f.FieldRange(pql.LT, bitDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{5000, 6000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{5000, 6000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with set bit). + // Query for fields less than or equal to (ending with set column). if b, err := f.FieldRange(pql.LTE, bitDepth, 301); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 4000, 5000, 6000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 4000, 5000, 6000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields less than or equal to (ending with unset bit). + // Query for fields less than or equal to (ending with unset column). if b, err := f.FieldRange(pql.LTE, bitDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{2000, 5000, 6000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{2000, 5000, 6000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) @@ -451,29 +451,29 @@ func TestFragment_FieldRange(t *testing.T) { // Query for fields greater than (ending with unset bit). if b, err := f.FieldRange(pql.GT, bitDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than (ending with set bit). if b, err := f.FieldRange(pql.GT, bitDepth, 301); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with unset bit). if b, err := f.FieldRange(pql.GTE, bitDepth, 300); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } // Query for fields greater than or equal to (ending with set bit). if b, err := f.FieldRange(pql.GTE, bitDepth, 301); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) @@ -496,32 +496,32 @@ func TestFragment_FieldRange(t *testing.T) { t.Fatal(err) } - // Query for fields greater than (ending with unset bit). + // Query for fields greater than (ending with unset column). if b, err := f.FieldRangeBetween(bitDepth, 300, 2817); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than (ending with set bit). + // Query for fields greater than (ending with set column). if b, err := f.FieldRangeBetween(bitDepth, 301, 2817); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 3000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 3000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with unset bit). + // Query for fields greater than or equal to (ending with unset column). if b, err := f.FieldRangeBetween(bitDepth, 301, 2816); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } - // Query for fields greater than or equal to (ending with set bit). + // Query for fields greater than or equal to (ending with set column). if b, err := f.FieldRangeBetween(bitDepth, 300, 2816); err != nil { t.Fatal(err) - } else if !reflect.DeepEqual(b.Bits(), []uint64{1000, 2000, 4000}) { - t.Fatalf("unexpected bits: %+v", b.Bits()) + } else if !reflect.DeepEqual(b.Columns(), []uint64{1000, 2000, 4000}) { + t.Fatalf("unexpected columns: %+v", b.Columns()) } }) } @@ -663,7 +663,7 @@ func TestFragment_TopN_Intersect(t *testing.T) { } } -// Ensure a fragment can return top rows that have many bits set. +// Ensure a fragment can return top rows that have many columns set. func TestFragment_TopN_Intersect_Large(t *testing.T) { if testing.Short() { t.Skip("short mode") @@ -1011,8 +1011,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { } // Verify data in other fragment. - if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { - t.Fatalf("unexpected bits: %+v", a) + if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + t.Fatalf("unexpected columns: %+v", a) } // Close and reopen the fragment & verify the data. @@ -1020,8 +1020,8 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) { t.Fatal(err) } else if n := f1.Cache().Len(); n != 1 { t.Fatalf("unexpected cache size (reopen): %d", n) - } else if a := f1.Row(1000).Bits(); !reflect.DeepEqual(a, []uint64{2}) { - t.Fatalf("unexpected bits (reopen): %+v", a) + } else if a := f1.Row(1000).Columns(); !reflect.DeepEqual(a, []uint64{2}) { + t.Fatalf("unexpected columns (reopen): %+v", a) } } diff --git a/frame.go b/frame.go index de9d3527f..ab005edcc 100644 --- a/frame.go +++ b/frame.go @@ -31,8 +31,7 @@ import ( // Default frame settings. const ( - DefaultCacheType = CacheTypeRanked - DefaultInverseEnabled = false + DefaultCacheType = CacheTypeRanked // Default ranked frame cache DefaultCacheSize = 50000 @@ -54,11 +53,10 @@ type Frame struct { Stats StatsClient // Frame options. - inverseEnabled bool - cacheType string - cacheSize uint32 - timeQuantum TimeQuantum - fields []*Field + cacheType string + cacheSize uint32 + timeQuantum TimeQuantum + fields []*Field Logger Logger } @@ -82,9 +80,8 @@ func NewFrame(path, index, name string) (*Frame, error) { broadcaster: NopBroadcaster, Stats: NopStatsClient, - inverseEnabled: DefaultInverseEnabled, - cacheType: DefaultCacheType, - cacheSize: DefaultCacheSize, + cacheType: DefaultCacheType, + cacheSize: DefaultCacheSize, //timeQuantum //fields @@ -111,37 +108,18 @@ func (f *Frame) MaxSlice() uint64 { var max uint64 for _, view := range f.views { - if view.name == ViewInverse { - continue - } else if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max { + if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max { max = viewMaxSlice } } return max } -// MaxInverseSlice returns the max inverse slice in the frame. -func (f *Frame) MaxInverseSlice() uint64 { - f.mu.RLock() - defer f.mu.RUnlock() - - view := f.views[ViewInverse] - if view == nil { - return 0 - } - return view.MaxSlice() -} - // CacheType returns the caching mode for the frame. func (f *Frame) CacheType() string { return f.cacheType } -// InverseEnabled returns true if an inverse view is available. -func (f *Frame) InverseEnabled() bool { - return f.inverseEnabled -} - // SetCacheSize sets the cache size for ranked fames. Persists to meta file on update. // defaults to DefaultCacheSize 50000 func (f *Frame) SetCacheSize(v uint32) error { @@ -179,11 +157,10 @@ func (f *Frame) Options() FrameOptions { func (f *Frame) options() FrameOptions { return FrameOptions{ - InverseEnabled: f.inverseEnabled, - CacheType: f.cacheType, - CacheSize: f.cacheSize, - TimeQuantum: f.timeQuantum, - Fields: f.fields, + CacheType: f.cacheType, + CacheSize: f.cacheSize, + TimeQuantum: f.timeQuantum, + Fields: f.fields, } } @@ -255,7 +232,6 @@ func (f *Frame) loadMeta() error { // Read data from meta file. buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta")) if os.IsNotExist(err) { - f.inverseEnabled = DefaultInverseEnabled f.cacheType = DefaultCacheType f.cacheSize = DefaultCacheSize f.timeQuantum = "" @@ -270,7 +246,6 @@ func (f *Frame) loadMeta() error { } // Copy metadata fields. - f.inverseEnabled = pb.InverseEnabled f.cacheType = pb.CacheType if f.cacheType == "" { f.cacheType = DefaultCacheType @@ -532,11 +507,6 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) { // createViewIfNotExistsBase returns the named view, creating it if necessary. // The returned bool indicates whether the view was created or not. func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) { - // Don't create inverse views if they are not enabled. - if !f.InverseEnabled() && IsInverseView(name) { - return nil, false, ErrFrameInverseDisabled - } - f.mu.Lock() defer f.mu.Unlock() @@ -840,16 +810,14 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro timestamp = timestamps[i] } - var standard, inverse []string + var standard []string if timestamp == nil { standard = []string{ViewStandard} - inverse = []string{ViewInverse} } else { standard = ViewsByTime(ViewStandard, *timestamp, q) // In order to match the logic of `SetBit()`, we want bits // with timestamps to write to both time and standard views. standard = append(standard, ViewStandard) - inverse = ViewsByTime(ViewInverse, *timestamp, q) } // Attach bit to each standard view. @@ -860,34 +828,10 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro data.ColumnIDs = append(data.ColumnIDs, columnID) dataByFragment[key] = data } - - if f.inverseEnabled { - // Attach reversed bits to each inverse view. - for _, name := range inverse { - key := importKey{View: name, Slice: rowID / SliceWidth} - data := dataByFragment[key] - data.RowIDs = append(data.RowIDs, columnID) // reversed - data.ColumnIDs = append(data.ColumnIDs, rowID) // reversed - dataByFragment[key] = data - } - } } // Import into each fragment. for key, data := range dataByFragment { - // Skip inverse data if inverse is not enabled. - if !f.inverseEnabled && IsInverseView(key.View) { - continue - } - - // Re-sort data for inverse views. - if IsInverseView(key.View) { - sort.Sort(importBitSet{ - rowIDs: data.RowIDs, - columnIDs: data.ColumnIDs, - }) - } - view, err := f.CreateViewIfNotExists(key.View) if err != nil { return errors.Wrap(err, "creating view") @@ -1003,12 +947,10 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name } // FrameOptions represents options to set when initializing a frame. type FrameOptions struct { - InverseEnabled bool `json:"inverseEnabled,omitempty"` - RangeEnabled bool `json:"rangeEnabled,omitempty"` // deprecated, will be removed - CacheType string `json:"cacheType,omitempty"` - CacheSize uint32 `json:"cacheSize,omitempty"` - TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` - Fields []*Field `json:"fields,omitempty"` + CacheType string `json:"cacheType,omitempty"` + CacheSize uint32 `json:"cacheSize,omitempty"` + TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"` + Fields []*Field `json:"fields,omitempty"` } // Encode converts o into its internal representation. @@ -1021,11 +963,10 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta { return nil } return &internal.FrameMeta{ - InverseEnabled: o.InverseEnabled, - CacheType: o.CacheType, - CacheSize: o.CacheSize, - TimeQuantum: string(o.TimeQuantum), - Fields: encodeFields(o.Fields), + CacheType: o.CacheType, + CacheSize: o.CacheSize, + TimeQuantum: string(o.TimeQuantum), + Fields: encodeFields(o.Fields), } } @@ -1034,11 +975,10 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions { return nil } return &FrameOptions{ - InverseEnabled: options.InverseEnabled, - CacheType: options.CacheType, - CacheSize: options.CacheSize, - TimeQuantum: TimeQuantum(options.TimeQuantum), - Fields: decodeFields(options.Fields), + CacheType: options.CacheType, + CacheSize: options.CacheSize, + TimeQuantum: TimeQuantum(options.TimeQuantum), + Fields: decodeFields(options.Fields), } } diff --git a/handler.go b/handler.go index a11c5b43f..256e3411e 100644 --- a/handler.go +++ b/handler.go @@ -81,8 +81,8 @@ func NewHandler() *Handler { func (h *Handler) populateValidators() { h.validators = map[string]*queryValidationSpec{} h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index") - h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse") - h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeAttrs", "excludeBits") + h.validators["GetSliceMax"] = queryValidationSpecRequired() + h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns") h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice") h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") h.validators["PostFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice") @@ -119,7 +119,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST") router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET") router.Handle("/debug/vars", expvar.Handler()).Methods("GET") - router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData") router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET") router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups) router.HandleFunc("/status", handler.handleGetStatus).Methods("GET") @@ -134,7 +133,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport") router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET") router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks") - router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST").Name("PostFragmentData") router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes") router.HandleFunc("/import", handler.handlePostImport).Methods("POST") router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST") @@ -147,7 +145,6 @@ func NewRouter(handler *Handler) *mux.Router { router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE") router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST") - router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST") router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET") router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE") @@ -303,7 +300,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) { func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{ Standard: h.API.MaxSlices(r.Context()), - Inverse: h.API.MaxInverseSlices(r.Context()), }); err != nil { h.Logger.Printf("write slices-max response error: %s", err) } @@ -311,7 +307,6 @@ func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) { type getSlicesMaxResponse struct { Standard map[string]uint64 `json:"standard"` - Inverse map[string]uint64 `json:"inverse"` } // handleGetIndexes handles GET /index request. @@ -821,11 +816,11 @@ func (h *Handler) readURLQueryRequest(r *http.Request) (*QueryRequest, error) { } return &QueryRequest{ - Query: query, - Slices: slices, - ColumnAttrs: q.Get("columnAttrs") == "true", - ExcludeAttrs: q.Get("excludeAttrs") == "true", - ExcludeBits: q.Get("excludeBits") == "true", + Query: query, + Slices: slices, + ColumnAttrs: q.Get("columnAttrs") == "true", + ExcludeRowAttrs: q.Get("excludeRowAttrs") == "true", + ExcludeColumns: q.Get("excludeColumns") == "true", }, nil } @@ -1017,48 +1012,6 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request) } } -// handleGetFragmentData handles GET /fragment/data requests. -func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) { - // Read slice parameter. - q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) - return - } - - // Retrieve fragment from holder. - f, err := h.API.MarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice) - if err != nil { - http.Error(w, err.Error(), http.StatusNotFound) - return - } - - // Stream fragment to response body. - if _, err := f.WriteTo(w); err != nil { - h.Logger.Printf("fragment backup error: %s", err) - } -} - -// handlePostFragmentData handles POST /fragment/data requests. -func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) { - // Read slice parameter. - q := r.URL.Query() - slice, err := strconv.ParseUint(q.Get("slice"), 10, 64) - if err != nil { - http.Error(w, "slice required", http.StatusBadRequest) - return - } - - if err = h.API.UnmarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil { - if errors.Cause(err) == ErrFrameNotFound { - http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound) - } else { - http.Error(w, err.Error(), http.StatusInternalServerError) - } - } -} - // handleGetFragmentBlockData handles GET /fragment/block/data requests. func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) { buf, err := h.API.FragmentBlockData(r.Context(), r.Body) @@ -1111,38 +1064,6 @@ type getFragmentBlocksResponse struct { Blocks []FragmentBlock `json:"blocks"` } -// handlePostFrameRestore handles POST /frame/restore requests. -func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) { - indexName := mux.Vars(r)["index"] - frameName := mux.Vars(r)["frame"] - - q := r.URL.Query() - hostStr := q.Get("host") - - // Validate query parameters. - if hostStr == "" { - http.Error(w, "host required", http.StatusBadRequest) - return - } - - host, err := NewURIFromAddress(hostStr) - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - } - - err = h.API.RestoreFrame(r.Context(), indexName, frameName, host) - switch errors.Cause(err) { - case nil: - break - case ErrFrameNotFound: - fallthrough - case ErrFragmentNotFound: - http.Error(w, err.Error(), http.StatusNotFound) - default: - http.Error(w, err.Error(), http.StatusInternalServerError) - } -} - // handleGetVersion handles /version requests. func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) { err := json.NewEncoder(w).Encode(struct { @@ -1181,10 +1102,10 @@ type QueryRequest struct { ColumnAttrs bool // Do not return row attributes, if true. - ExcludeAttrs bool + ExcludeRowAttrs bool - // Do not return bits, if true. - ExcludeBits bool + // Do not return columns, if true. + ExcludeColumns bool // If true, indicates that query is part of a larger distributed query. // If false, this request is on the originating node. @@ -1193,12 +1114,12 @@ type QueryRequest struct { func decodeQueryRequest(pb *internal.QueryRequest) *QueryRequest { req := &QueryRequest{ - Query: pb.Query, - Slices: pb.Slices, - ColumnAttrs: pb.ColumnAttrs, - Remote: pb.Remote, - ExcludeAttrs: pb.ExcludeAttrs, - ExcludeBits: pb.ExcludeBits, + Query: pb.Query, + Slices: pb.Slices, + ColumnAttrs: pb.ColumnAttrs, + Remote: pb.Remote, + ExcludeRowAttrs: pb.ExcludeRowAttrs, + ExcludeColumns: pb.ExcludeColumns, } return req diff --git a/handler_internal_test.go b/handler_internal_test.go index 4603662e6..ceaeb707c 100644 --- a/handler_internal_test.go +++ b/handler_internal_test.go @@ -66,8 +66,8 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) { {json: `{"options": 4}`, err: "options is not map[string]interface{}"}, {json: `{"option": {}}`, err: "Unknown key: option:map[]"}, {json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"}, - {json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}}, - {json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}}, + {json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"}, + {json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}}, {json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"}, } for _, test := range tests { diff --git a/handler_test.go b/handler_test.go index b39b207c5..24587aa0d 100644 --- a/handler_test.go +++ b/handler_test.go @@ -84,12 +84,10 @@ func TestHandler_Schema(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil { - t.Fatal(err) } if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { t.Fatal(err) @@ -107,8 +105,8 @@ func TestHandler_Schema(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { - } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -123,12 +121,10 @@ func TestHandler_Status(t *testing.T) { i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}) i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}) - if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil { + if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil { t.Fatal(err) } else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil { t.Fatal(err) - } else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil { - t.Fatal(err) } if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil { t.Fatal(err) @@ -209,48 +205,7 @@ func TestHandler_MaxSlices(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil)) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" { - t.Fatalf("unexpected body: %s", body) - } -} - -// Ensure the handler can return the maxslice map for the inverse views. -func TestHandler_MaxSlices_Inverse(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true}) - if err != nil { - t.Fatal(err) - } - if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+1, nil); err != nil { - t.Fatal(err) - } else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+2, nil); err != nil { - t.Fatal(err) - } else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (3*SliceWidth)+4, nil); err != nil { - t.Fatal(err) - } - - f1, err := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true}) - if err != nil { - t.Fatal(err) - } - if _, err := f1.SetBit(pilosa.ViewStandard, 40, (0*SliceWidth)+1, nil); err != nil { - t.Fatal(err) - } else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+2, nil); err != nil { - t.Fatal(err) - } else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+4, nil); err != nil { - t.Fatal(err) - } - - h := test.NewHandler() - h.API.Holder = hldr.Holder - h.API.Cluster = test.NewCluster(1) - w := httptest.NewRecorder() - h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil)) - if w.Code != http.StatusOK { - t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" { + } else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -419,7 +374,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -452,7 +407,7 @@ func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) { h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)"))) if w.Code != http.StatusOK { t.Fatalf("unexpected status code: %d", w.Code) - } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { + } else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" { t.Fatalf("unexpected body: %s", body) } } @@ -484,8 +439,8 @@ func TestHandler_Query_Row_Protobuf(t *testing.T) { t.Fatal(err) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) - } else if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + } else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { t.Fatalf("unexpected attr length: %d", len(attrs)) } else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" { @@ -541,8 +496,8 @@ func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) { if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil { t.Fatal(err) } - if bits := resp.Results[0].Row.Bits; !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 1}) { - t.Fatalf("unexpected bits: %+v", bits) + if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, SliceWidth + 1}) { + t.Fatalf("unexpected columns: %+v", columns) } else if rt := resp.Results[0].Type; rt != pilosa.QueryResultTypeRow { t.Fatalf("unexpected response type: %d", resp.Results[0].Type) } else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 { @@ -1101,55 +1056,6 @@ type FrameFields struct { Fields []pilosa.Field } -// Ensure the handler can backup a fragment and then restore it. -func TestHandler_Fragment_BackupRestore(t *testing.T) { - hldr := test.MustOpenHolder() - defer hldr.Close() - - s := test.NewServer() - s.Handler.API.Holder = hldr.Holder - defer s.Close() - - // Set bits in the index. - f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0) - f0.MustSetBits(100, 1, 2, 3) - - // Begin backing up from slice i/f/0. - resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0") - if err != nil { - t.Fatal(err) - } - defer resp.Body.Close() - - // Ensure response came back OK. - if resp.StatusCode != http.StatusOK { - t.Fatalf("unexpected backup status code: %d", resp.StatusCode) - } - - // Create frame. - if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil { - t.Fatal(err) - } - - // Restore backup to slice x/y/0. - if resp, err := http.Post(s.URL+"/fragment/data?index=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil { - t.Fatal(err) - } else if resp.StatusCode != http.StatusOK { - resp.Body.Close() - t.Fatalf("unexpected restore status code: %d", resp.StatusCode) - } else { - resp.Body.Close() - } - - // Verify data is correctly restored. - f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0) - if f1 == nil { - t.Fatal("fragment x/y/standard/0 not created") - } else if bits := f1.Row(100).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 3}) { - t.Fatalf("unexpected restored bits: %+v", bits) - } -} - // Ensure the handler can retrieve the version. func TestHandler_Version(t *testing.T) { hldr := test.MustOpenHolder() diff --git a/holder.go b/holder.go index a80ba4539..6d13159b9 100644 --- a/holder.go +++ b/holder.go @@ -209,15 +209,6 @@ func (h *Holder) MaxSlices() map[string]uint64 { return a } -// MaxInverseSlices returns MaxInverseSlice map for all indexes. -func (h *Holder) MaxInverseSlices() map[string]uint64 { - a := make(map[string]uint64) - for _, index := range h.Indexes() { - a[index.Name()] = index.MaxInverseSlice() - } - return a -} - // Schema returns schema information for all indexes, frames, and views. func (h *Holder) Schema() []*IndexInfo { var a []*IndexInfo @@ -270,7 +261,6 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error { func (h *Holder) EncodeMaxSlices() *internal.MaxSlices { return &internal.MaxSlices{ Standard: h.MaxSlices(), - Inverse: h.MaxInverseSlices(), } } diff --git a/holder_test.go b/holder_test.go index 419cdc3f2..c7877e3ba 100644 --- a/holder_test.go +++ b/holder_test.go @@ -456,29 +456,29 @@ func TestHolderSyncer_SyncHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0, hldr1} { f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected bits(%d/0): %+v", i, a) - } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected bits(%d/2): %+v", i, a) - } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/3): %+v", i, a) - } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/120): %+v", i, a) - } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected bits(%d/200): %+v", i, a) + if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) } f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Bits() + a := f.Row(9).Columns() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a) + t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) } - if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) + if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } f = hldr.Fragment("y", "z", pilosa.ViewStandard, 3) - if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { - t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(3 * SliceWidth) + 4, (3 * SliceWidth) + 5, (3 * SliceWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } } @@ -554,29 +554,29 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected bits(%d/0): %+v", i, a) - } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected bits(%d/2): %+v", i, a) - } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/3): %+v", i, a) - } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/120): %+v", i, a) - } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected bits(%d/200): %+v", i, a) + if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) } f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) - a := f.Row(9).Bits() + a := f.Row(9).Columns() if !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected bits(%d/i/f0): %+v", i, a) + t.Fatalf("unexpected columns(%d/i/f0): %+v", i, a) } - if a := f.Row(9).Bits(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { - t.Fatalf("unexpected bits(%d/d/f0): %+v", i, a) + if a := f.Row(9).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth + 5}) { + t.Fatalf("unexpected columns(%d/d/f0): %+v", i, a) } f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { - t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } @@ -597,16 +597,16 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { // Verify data is the same on both nodes. for i, hldr := range []*test.Holder{hldr0} { f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0) - if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) { - t.Fatalf("unexpected bits(%d/0): %+v", i, a) - } else if a := f.Row(2).Bits(); !reflect.DeepEqual(a, []uint64{20}) { - t.Fatalf("unexpected bits(%d/2): %+v", i, a) - } else if a := f.Row(3).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/3): %+v", i, a) - } else if a := f.Row(120).Bits(); !reflect.DeepEqual(a, []uint64{10}) { - t.Fatalf("unexpected bits(%d/120): %+v", i, a) - } else if a := f.Row(200).Bits(); !reflect.DeepEqual(a, []uint64{4}) { - t.Fatalf("unexpected bits(%d/200): %+v", i, a) + if a := f.Row(0).Columns(); !reflect.DeepEqual(a, []uint64{10, 4000}) { + t.Fatalf("unexpected columns(%d/0): %+v", i, a) + } else if a := f.Row(2).Columns(); !reflect.DeepEqual(a, []uint64{20}) { + t.Fatalf("unexpected columns(%d/2): %+v", i, a) + } else if a := f.Row(3).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/3): %+v", i, a) + } else if a := f.Row(120).Columns(); !reflect.DeepEqual(a, []uint64{10}) { + t.Fatalf("unexpected columns(%d/120): %+v", i, a) + } else if a := f.Row(200).Columns(); !reflect.DeepEqual(a, []uint64{4}) { + t.Fatalf("unexpected columns(%d/200): %+v", i, a) } f = hldr.Fragment("i", "f0", pilosa.ViewStandard, 1) @@ -615,8 +615,8 @@ func TestHolderCleaner_CleanHolder(t *testing.T) { } f = hldr.Fragment("y", "z", pilosa.ViewStandard, 2) - if a := f.Row(10).Bits(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { - t.Fatalf("unexpected bits(%d/y/z): %+v", i, a) + if a := f.Row(10).Columns(); !reflect.DeepEqual(a, []uint64{(2 * SliceWidth) + 4, (2 * SliceWidth) + 5, (2 * SliceWidth) + 7}) { + t.Fatalf("unexpected columns(%d/y/z): %+v", i, a) } } } diff --git a/index.go b/index.go index 4212576f6..534930cd5 100644 --- a/index.go +++ b/index.go @@ -38,8 +38,7 @@ type Index struct { frames map[string]*Frame // Max Slice on any node in the cluster, according to this node. - remoteMaxSlice uint64 - remoteMaxInverseSlice uint64 + remoteMaxSlice uint64 NewAttrStore func(string) AttrStore @@ -64,8 +63,7 @@ func NewIndex(path, name string) (*Index, error) { name: name, frames: make(map[string]*Frame), - remoteMaxSlice: 0, - remoteMaxInverseSlice: 0, + remoteMaxSlice: 0, NewAttrStore: NewNopAttrStore, columnAttrStore: NopAttrStore, @@ -236,30 +234,6 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) { i.remoteMaxSlice = newmax } -// MaxInverseSlice returns the max inverse slice in the index according to this node. -func (i *Index) MaxInverseSlice() uint64 { - if i == nil { - return 0 - } - i.mu.RLock() - defer i.mu.RUnlock() - - max := i.remoteMaxInverseSlice - for _, f := range i.frames { - if slice := f.MaxInverseSlice(); slice > max { - max = slice - } - } - return max -} - -// SetRemoteMaxInverseSlice sets the remote max inverse slice value received from another node. -func (i *Index) SetRemoteMaxInverseSlice(v uint64) { - i.mu.Lock() - defer i.mu.Unlock() - i.remoteMaxInverseSlice = v -} - // FramePath returns the path to a frame in the index. func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) } @@ -325,11 +299,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { return nil, ErrInvalidCacheType } - // Validate mutually exclusive options if ranges are enabled. - if opt.RangeEnabled { - i.Logger.Printf("RangeEnabled is deprecated - no need to set RangeEnabled to true when creating a frame") - } - // Validate fields. for _, field := range opt.Fields { if err := ValidateField(field); err != nil { @@ -364,8 +333,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) { f.cacheSize = opt.CacheSize } - f.inverseEnabled = opt.InverseEnabled - // Set fields. f.fields = opt.Fields diff --git a/index_test.go b/index_test.go index a83fb7609..0a42804e2 100644 --- a/index_test.go +++ b/index_test.go @@ -67,14 +67,13 @@ func TestIndex_CreateFrame(t *testing.T) { }) // Ensure frame can include range columns. - t.Run("RangeEnabled", func(t *testing.T) { + t.Run("BSIFields", func(t *testing.T) { t.Run("OK", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() // Create frame with schema and verify it exists. if f, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: false, Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 10, Max: 20}, {Name: "field1", Type: pilosa.FieldTypeInt, Min: 11, Max: 21}, @@ -99,49 +98,6 @@ func TestIndex_CreateFrame(t *testing.T) { } }) - t.Run("ErrInverseRangeAllowed", func(t *testing.T) { - index := test.MustOpenIndex() - defer index.Close() - - frame, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, - InverseEnabled: true, - Fields: []*pilosa.Field{ - &pilosa.Field{ - Name: "myfield", - Type: pilosa.FieldTypeInt, - Min: -20, - Max: 100, - }, - }, - }) - if err != nil { - t.Fatal(err) - } - - ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil) - if !ch || err != nil { - t.Fatal(ch, err) - } - ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil) - if !ch || err != nil { - t.Fatal(ch, err) - } - ch, err = frame.SetFieldValue(1, "myfield", 87) - if !ch || err != nil { - t.Fatal(ch, err) - } - views := frame.Views() - if len(views) != 3 { - var names string - for _, v := range views { - names = names + v.Name() + " " - } - t.Fatalf("Unexpected views: %s", names) - } - - }) - t.Run("ErrRangeCacheAllowed", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() @@ -153,7 +109,7 @@ func TestIndex_CreateFrame(t *testing.T) { } }) - t.Run("RangeEnabledWithCacheTypeNone", func(t *testing.T) { + t.Run("BSIFieldsWithCacheTypeNone", func(t *testing.T) { index := test.MustOpenIndex() defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ @@ -208,7 +164,6 @@ func TestIndex_CreateFrame(t *testing.T) { defer index.Close() if _, err := index.CreateFrame("f", pilosa.FrameOptions{ - RangeEnabled: true, // make sure we can still create frames with RangeEnabled: true after deprecation Fields: []*pilosa.Field{ {Name: "field0", Type: pilosa.FieldTypeInt, Min: 100, Max: 50}, }, diff --git a/internal/private.pb.go b/internal/private.pb.go index d7922711d..35b452dce 100644 --- a/internal/private.pb.go +++ b/internal/private.pb.go @@ -70,12 +70,10 @@ func (*IndexMeta) ProtoMessage() {} func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} } type FrameMeta struct { - InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"` - CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` - CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` - TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` - RangeEnabled bool `protobuf:"varint,6,opt,name=RangeEnabled,proto3" json:"RangeEnabled,omitempty"` - Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"` + CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"` + CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"` + TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"` + Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"` } func (m *FrameMeta) Reset() { *m = FrameMeta{} } @@ -83,13 +81,6 @@ func (m *FrameMeta) String() string { return proto.CompactTextString( func (*FrameMeta) ProtoMessage() {} func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} } -func (m *FrameMeta) GetInverseEnabled() bool { - if m != nil { - return m.InverseEnabled - } - return false -} - func (m *FrameMeta) GetCacheType() string { if m != nil { return m.CacheType @@ -111,13 +102,6 @@ func (m *FrameMeta) GetTimeQuantum() string { return "" } -func (m *FrameMeta) GetRangeEnabled() bool { - if m != nil { - return m.RangeEnabled - } - return false -} - func (m *FrameMeta) GetFields() []*Field { if m != nil { return m.Fields @@ -231,7 +215,6 @@ func (m *Cache) GetIDs() []uint64 { type MaxSlices struct { Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` - Inverse map[string]uint64 `protobuf:"bytes,2,rep,name=Inverse" json:"Inverse,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"` } func (m *MaxSlices) Reset() { *m = MaxSlices{} } @@ -246,17 +229,9 @@ func (m *MaxSlices) GetStandard() map[string]uint64 { return nil } -func (m *MaxSlices) GetInverse() map[string]uint64 { - if m != nil { - return m.Inverse - } - return nil -} - type CreateSliceMessage struct { - Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` - Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` - IsInverse bool `protobuf:"varint,3,opt,name=IsInverse,proto3" json:"IsInverse,omitempty"` + Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` + Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"` } func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} } @@ -278,13 +253,6 @@ func (m *CreateSliceMessage) GetSlice() uint64 { return 0 } -func (m *CreateSliceMessage) GetIsInverse() bool { - if m != nil { - return m.IsInverse - } - return false -} - type DeleteIndexMessage struct { Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"` } @@ -1067,16 +1035,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if m.InverseEnabled { - dAtA[i] = 0x10 - i++ - if m.InverseEnabled { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } if len(m.CacheType) > 0 { dAtA[i] = 0x1a i++ @@ -1094,16 +1052,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(len(m.TimeQuantum))) i += copy(dAtA[i:], m.TimeQuantum) } - if m.RangeEnabled { - dAtA[i] = 0x30 - i++ - if m.RangeEnabled { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } if len(m.Fields) > 0 { for _, msg := range m.Fields { dAtA[i] = 0x3a @@ -1307,22 +1255,6 @@ func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintPrivate(dAtA, i, uint64(v)) } } - if len(m.Inverse) > 0 { - for k, _ := range m.Inverse { - dAtA[i] = 0x12 - i++ - v := m.Inverse[k] - mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) - i = encodeVarintPrivate(dAtA, i, uint64(mapSize)) - dAtA[i] = 0xa - i++ - i = encodeVarintPrivate(dAtA, i, uint64(len(k))) - i += copy(dAtA[i:], k) - dAtA[i] = 0x10 - i++ - i = encodeVarintPrivate(dAtA, i, uint64(v)) - } - } return i, nil } @@ -1352,16 +1284,6 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) { i++ i = encodeVarintPrivate(dAtA, i, uint64(m.Slice)) } - if m.IsInverse { - dAtA[i] = 0x18 - i++ - if m.IsInverse { - dAtA[i] = 1 - } else { - dAtA[i] = 0 - } - i++ - } return i, nil } @@ -2324,9 +2246,6 @@ func (m *IndexMeta) Size() (n int) { func (m *FrameMeta) Size() (n int) { var l int _ = l - if m.InverseEnabled { - n += 2 - } l = len(m.CacheType) if l > 0 { n += 1 + l + sovPrivate(uint64(l)) @@ -2338,9 +2257,6 @@ func (m *FrameMeta) Size() (n int) { if l > 0 { n += 1 + l + sovPrivate(uint64(l)) } - if m.RangeEnabled { - n += 2 - } if len(m.Fields) > 0 { for _, e := range m.Fields { l = e.Size() @@ -2428,14 +2344,6 @@ func (m *MaxSlices) Size() (n int) { n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) } } - if len(m.Inverse) > 0 { - for k, v := range m.Inverse { - _ = k - _ = v - mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v)) - n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) - } - } return n } @@ -2449,9 +2357,6 @@ func (m *CreateSliceMessage) Size() (n int) { if m.Slice != 0 { n += 1 + sovPrivate(uint64(m.Slice)) } - if m.IsInverse { - n += 2 - } return n } @@ -2957,26 +2862,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.InverseEnabled = bool(v != 0) case 3: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field CacheType", wireType) @@ -3054,26 +2939,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error { } m.TimeQuantum = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field RangeEnabled", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.RangeEnabled = bool(v != 0) case 7: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field Fields", wireType) @@ -3802,113 +3667,6 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error { } m.Standard[mapkey] = mapvalue iNdEx = postIndex - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Inverse", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPrivate - } - postIndex := iNdEx + msglen - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Inverse == nil { - m.Inverse = make(map[string]uint64) - } - var mapkey string - var mapvalue uint64 - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPrivate - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - mapvalue |= (uint64(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - } else { - iNdEx = entryPreIndex - skippy, err := skipPrivate(dAtA[iNdEx:]) - if err != nil { - return err - } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) > postIndex { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - m.Inverse[mapkey] = mapvalue - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -4007,26 +3765,6 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error { break } } - case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field IsInverse", wireType) - } - var v int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPrivate - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - v |= (int(b) & 0x7F) << shift - if b < 0x80 { - break - } - } - m.IsInverse = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPrivate(dAtA[iNdEx:]) @@ -7259,75 +6997,70 @@ var ( func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) } var fileDescriptorPrivate = []byte{ - // 1112 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0xcd, 0x6f, 0x1b, 0x45, - 0x14, 0x67, 0xbd, 0x6b, 0x27, 0x7e, 0xae, 0x53, 0x67, 0x5a, 0xca, 0xb6, 0xaa, 0x82, 0x19, 0x15, - 0x6a, 0x38, 0x44, 0x25, 0xbd, 0x40, 0xa1, 0x52, 0x95, 0x38, 0x15, 0x8b, 0x48, 0x04, 0xe3, 0xa4, - 0x07, 0x24, 0x90, 0x26, 0xf6, 0x28, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0xf9, 0xe8, 0x81, 0x33, - 0x17, 0xee, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x01, 0x42, 0xe1, 0x0f, 0x01, 0xbd, 0x37, 0xb3, 0x1f, - 0xb1, 0x9d, 0xa6, 0x04, 0x6e, 0xf3, 0x3e, 0xe7, 0xf7, 0x3e, 0x67, 0x17, 0xda, 0x93, 0x34, 0x3c, - 0x96, 0x5a, 0xad, 0x4f, 0xd2, 0x44, 0x27, 0x6c, 0x39, 0x8c, 0xb5, 0x4a, 0x63, 0x19, 0xf1, 0x16, - 0x34, 0x83, 0x78, 0xa4, 0x4e, 0x77, 0x94, 0x96, 0xfc, 0x0f, 0x07, 0x9a, 0xcf, 0x53, 0x39, 0x56, - 0x48, 0xb1, 0x0f, 0x60, 0x25, 0x88, 0x8f, 0x55, 0x9a, 0xa9, 0xed, 0x58, 0x1e, 0x44, 0x6a, 0xe4, - 0xd7, 0xba, 0x4e, 0x6f, 0x59, 0xcc, 0x70, 0xd9, 0x7d, 0x68, 0x6e, 0xc9, 0xe1, 0x4b, 0xb5, 0x77, - 0x36, 0x51, 0xbe, 0xdb, 0x75, 0x7a, 0x4d, 0x51, 0x32, 0x0a, 0xe9, 0x20, 0x7c, 0xa5, 0x7c, 0xaf, - 0xeb, 0xf4, 0xda, 0xa2, 0x64, 0xb0, 0x2e, 0xb4, 0xf6, 0xc2, 0xb1, 0xfa, 0x66, 0x2a, 0x63, 0x3d, - 0x1d, 0xfb, 0x75, 0xb2, 0xae, 0xb2, 0x18, 0x87, 0x1b, 0x42, 0xc6, 0x87, 0x05, 0x86, 0x06, 0x61, - 0xb8, 0xc0, 0x63, 0x0f, 0xa1, 0xf1, 0x3c, 0x54, 0xd1, 0x28, 0xf3, 0x97, 0xba, 0x6e, 0xaf, 0xb5, - 0x71, 0x73, 0x3d, 0x8f, 0x6f, 0x9d, 0xf8, 0xc2, 0x8a, 0x39, 0x87, 0x95, 0x60, 0x3c, 0x49, 0x52, - 0x2d, 0x54, 0x36, 0x49, 0xe2, 0x4c, 0xb1, 0x0e, 0xb8, 0xdb, 0x69, 0xea, 0x3b, 0x74, 0x31, 0x1e, - 0xf9, 0x8f, 0xd0, 0xd9, 0x8c, 0x92, 0xe1, 0x51, 0x5f, 0x6a, 0x29, 0xd4, 0x0f, 0x53, 0x95, 0x69, - 0x76, 0x1b, 0xea, 0x94, 0x25, 0xab, 0x67, 0x08, 0xe4, 0x52, 0xb6, 0x28, 0x2f, 0x4d, 0x61, 0x08, - 0xe4, 0x92, 0x3d, 0xa5, 0xc2, 0x13, 0x86, 0x40, 0xee, 0x20, 0x0a, 0x87, 0x26, 0x05, 0x9e, 0x30, - 0x04, 0x63, 0xe0, 0xbd, 0x08, 0xd5, 0x89, 0x8d, 0x9b, 0xce, 0x3c, 0x80, 0xd5, 0xca, 0xfd, 0x16, - 0xe6, 0x1d, 0x68, 0x88, 0xe4, 0x24, 0xe8, 0x67, 0xbe, 0xd3, 0x75, 0x7b, 0x9e, 0xb0, 0x14, 0x65, - 0x37, 0x89, 0xa6, 0xe3, 0x18, 0x45, 0x35, 0x12, 0x95, 0x0c, 0x7e, 0x17, 0xea, 0x94, 0x6a, 0x8c, - 0xb2, 0xb4, 0xc5, 0x23, 0xff, 0xdb, 0x81, 0xe6, 0x8e, 0x3c, 0x25, 0x18, 0x19, 0x7b, 0x0a, 0xcb, - 0x03, 0x2d, 0xe3, 0x91, 0x4c, 0x47, 0xa4, 0xd4, 0xda, 0x78, 0xaf, 0x4c, 0x61, 0xa1, 0xb6, 0x9e, - 0xeb, 0x6c, 0xc7, 0x3a, 0x3d, 0x13, 0x85, 0x09, 0x7b, 0x02, 0x4b, 0xb6, 0x27, 0x08, 0x43, 0x6b, - 0xa3, 0xbb, 0xc8, 0xba, 0x68, 0x1b, 0x34, 0xce, 0x0d, 0xee, 0x7d, 0x06, 0xed, 0x0b, 0x6e, 0x11, - 0xeb, 0x91, 0x3a, 0xcb, 0x2b, 0x72, 0xa4, 0xce, 0x30, 0x77, 0xc7, 0x32, 0x9a, 0x9a, 0x3c, 0x7b, - 0xc2, 0x10, 0x4f, 0x6a, 0x9f, 0x38, 0xf7, 0x9e, 0xc0, 0x8d, 0xaa, 0xd7, 0x7f, 0x63, 0xcb, 0xbf, - 0x07, 0xb6, 0x95, 0x2a, 0xa9, 0x15, 0xc1, 0xdb, 0x51, 0x59, 0x26, 0x0f, 0xd5, 0xe5, 0x95, 0x36, - 0xd5, 0xab, 0x55, 0xab, 0x77, 0x1f, 0x9a, 0x41, 0x96, 0x07, 0xee, 0x52, 0x5f, 0x96, 0x0c, 0xfe, - 0x11, 0xb0, 0xbe, 0x8a, 0x94, 0x56, 0x76, 0xbe, 0x5e, 0xe3, 0x9f, 0x0f, 0x72, 0x2c, 0x57, 0xeb, - 0xb2, 0x87, 0xe0, 0xe1, 0x78, 0x12, 0x94, 0xd6, 0xc6, 0xad, 0x32, 0xd3, 0xc5, 0x1c, 0x0b, 0x52, - 0xe0, 0x61, 0xee, 0xd4, 0x8e, 0xf4, 0x15, 0x01, 0x2e, 0x68, 0xe5, 0xfc, 0x2a, 0x77, 0xf6, 0xaa, - 0x62, 0x49, 0xd8, 0xab, 0x9e, 0xe5, 0xb1, 0x5e, 0xf7, 0x2a, 0x7e, 0x58, 0x80, 0xc5, 0x49, 0xbd, - 0x0e, 0xd8, 0xf7, 0xa1, 0x4e, 0xb6, 0x16, 0xed, 0xdc, 0x0e, 0x30, 0x52, 0xfe, 0xa2, 0x80, 0x7a, - 0xdd, 0x8b, 0x6e, 0x57, 0x2f, 0x6a, 0xe6, 0x7e, 0xbf, 0xb5, 0xba, 0x38, 0xd3, 0xbb, 0x68, 0x63, - 0x3c, 0xd1, 0xf9, 0xf2, 0x9a, 0xcd, 0x24, 0x12, 0x7d, 0xe3, 0x12, 0xc8, 0x7c, 0xb7, 0xeb, 0xa2, - 0x6f, 0x22, 0xf8, 0x63, 0x68, 0x0c, 0x86, 0x2f, 0xd5, 0x58, 0xb2, 0x0f, 0x71, 0xd2, 0x46, 0xea, - 0x54, 0x65, 0x76, 0x4e, 0x6f, 0xce, 0xd4, 0x5f, 0xe4, 0x72, 0xde, 0xb7, 0x21, 0x5d, 0x02, 0xa8, - 0x41, 0x57, 0x67, 0xbe, 0x37, 0xb7, 0x31, 0x91, 0x2f, 0xac, 0x98, 0x6f, 0x83, 0xbb, 0x2f, 0x02, - 0xdc, 0x3f, 0x84, 0x20, 0xf7, 0x62, 0x29, 0xf4, 0xfd, 0x45, 0x92, 0x69, 0x9b, 0x20, 0x3a, 0x23, - 0xef, 0xeb, 0x24, 0xd5, 0x94, 0x9e, 0xb6, 0xa0, 0x33, 0xff, 0x0e, 0xbc, 0xdd, 0x64, 0xa4, 0xd8, - 0x0a, 0xd4, 0x82, 0xbe, 0xf5, 0x51, 0x0b, 0xfa, 0xec, 0x5d, 0x72, 0x6f, 0xf3, 0xd2, 0x2e, 0x41, - 0xec, 0x8b, 0x40, 0xd0, 0xc5, 0x0f, 0xa0, 0x1d, 0x64, 0x5b, 0x49, 0x92, 0x8e, 0xc2, 0x58, 0xea, - 0x24, 0xb5, 0x73, 0x76, 0x91, 0xc9, 0x9f, 0x41, 0x07, 0xdd, 0x0f, 0xb4, 0xd4, 0x45, 0xf7, 0xdd, - 0x81, 0x06, 0xf2, 0x8a, 0xeb, 0x2c, 0x45, 0xb3, 0x8c, 0x7a, 0x79, 0x51, 0x89, 0xe0, 0x5f, 0x19, - 0x0f, 0xdb, 0xc7, 0x2a, 0xd6, 0x95, 0xa6, 0x20, 0x9a, 0x1c, 0xb4, 0x85, 0x21, 0x18, 0x37, 0xa1, - 0x58, 0xcc, 0x2b, 0x25, 0x66, 0xe4, 0x0a, 0x92, 0xf1, 0x9f, 0x1d, 0x80, 0x1c, 0xd0, 0x34, 0x2b, - 0x4c, 0x9c, 0xcb, 0x4d, 0xd8, 0xc7, 0x95, 0x7d, 0x3c, 0xdf, 0x27, 0x85, 0x48, 0x54, 0xb6, 0x76, - 0x2f, 0x6f, 0x0b, 0xdb, 0xf2, 0x9d, 0x52, 0xdf, 0xf0, 0x6d, 0x99, 0x70, 0x15, 0xb4, 0xb7, 0xa2, - 0x69, 0xa6, 0x55, 0x6a, 0x11, 0xe1, 0xbb, 0x61, 0x18, 0x45, 0x7e, 0x4a, 0xc6, 0xe2, 0x14, 0xb1, - 0x07, 0x50, 0x47, 0xa4, 0xa6, 0x37, 0xe7, 0xc3, 0x30, 0x42, 0x3e, 0xb0, 0xd3, 0xb1, 0xb0, 0xed, - 0x18, 0x78, 0xf4, 0x95, 0x60, 0xdb, 0x85, 0x3e, 0x10, 0x3a, 0xe0, 0xee, 0x84, 0x31, 0x85, 0xe0, - 0x0a, 0x3c, 0x12, 0x47, 0x9e, 0xd2, 0x4b, 0x89, 0x1c, 0x89, 0xfb, 0x71, 0xd5, 0x6c, 0x07, 0x9c, - 0x87, 0xeb, 0xcc, 0x6c, 0xfe, 0xd0, 0xba, 0x95, 0x87, 0x76, 0x00, 0xab, 0x66, 0x13, 0xfc, 0x9f, - 0x4e, 0x7f, 0xad, 0xc1, 0xaa, 0x50, 0x59, 0xf8, 0x4a, 0x05, 0x71, 0xa6, 0xd3, 0xe9, 0x50, 0x87, - 0x49, 0x8c, 0xf6, 0x5f, 0x26, 0x07, 0x36, 0xd5, 0xae, 0x30, 0xc4, 0x9b, 0x74, 0x12, 0x7b, 0x04, - 0xad, 0xd9, 0xee, 0x9f, 0x57, 0xad, 0xaa, 0xb0, 0x47, 0xb0, 0x34, 0x48, 0xa6, 0xe9, 0xb0, 0x98, - 0xed, 0x3b, 0xa5, 0xb6, 0x41, 0x66, 0xc4, 0x22, 0x57, 0xab, 0xf4, 0x51, 0xfd, 0xf5, 0x7d, 0xc4, - 0x9e, 0xce, 0xf4, 0x11, 0x7d, 0x8d, 0xb5, 0x36, 0xde, 0x29, 0x0d, 0x2e, 0x88, 0xc5, 0x45, 0x6d, - 0xfe, 0x93, 0x03, 0x37, 0xaa, 0x10, 0xde, 0x68, 0x30, 0x8a, 0x8a, 0xd4, 0x16, 0x56, 0xc4, 0x5d, - 0x54, 0x11, 0xaf, 0xac, 0x48, 0xf9, 0x76, 0xd7, 0x2b, 0x6f, 0x37, 0x3f, 0x82, 0xbb, 0x73, 0x65, - 0xda, 0x4a, 0xc6, 0x13, 0xec, 0x87, 0xff, 0x50, 0x2e, 0x5c, 0x19, 0x69, 0x6a, 0x0b, 0xd5, 0x14, - 0x86, 0xe0, 0x9f, 0xc2, 0xdb, 0x03, 0xa5, 0x2b, 0x45, 0xca, 0xbb, 0xad, 0x0b, 0xee, 0xae, 0x3a, - 0xb9, 0x24, 0x7c, 0x14, 0xf1, 0xcf, 0xc1, 0xdf, 0x9f, 0x8c, 0xa4, 0x56, 0xd7, 0xb2, 0xde, 0x84, - 0xe5, 0xbd, 0x64, 0x92, 0x44, 0xc9, 0xe1, 0xd9, 0x15, 0x23, 0xef, 0xc3, 0x92, 0xd9, 0x8f, 0xe6, - 0x33, 0xb2, 0x29, 0x72, 0x92, 0xdf, 0xc2, 0x86, 0x1e, 0xca, 0x68, 0x38, 0x8d, 0x10, 0x06, 0x7e, - 0x4f, 0x66, 0x9b, 0x9d, 0xdf, 0xce, 0xd7, 0x9c, 0xdf, 0xcf, 0xd7, 0x9c, 0x3f, 0xcf, 0xd7, 0x9c, - 0x5f, 0xfe, 0x5a, 0x7b, 0xeb, 0xa0, 0x41, 0x7f, 0x16, 0x8f, 0xff, 0x09, 0x00, 0x00, 0xff, 0xff, - 0xfa, 0xf5, 0x5b, 0x36, 0x6a, 0x0c, 0x00, 0x00, + // 1035 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1b, 0x45, + 0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x8c, 0x53, 0x67, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45, + 0x0d, 0x1c, 0xa2, 0x92, 0x5e, 0x78, 0x55, 0x8a, 0x12, 0xa7, 0x62, 0x11, 0x89, 0x60, 0x36, 0xe9, + 0x01, 0x89, 0xc3, 0xd4, 0x1e, 0xa5, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x1e, 0x3d, 0x70, 0x85, + 0x0b, 0x17, 0x4e, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f, + 0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xfb, 0xbd, 0x5f, 0xbf, 0xef, 0x9b, 0x85, 0xee, 0x24, 0x8d, 0xce, + 0xb9, 0x12, 0xdb, 0x93, 0x54, 0x2a, 0x49, 0x56, 0xa3, 0x44, 0x89, 0x34, 0xe1, 0x31, 0xed, 0x40, + 0x3b, 0x48, 0x46, 0xe2, 0xf2, 0x50, 0x28, 0x4e, 0x7f, 0x77, 0xa0, 0xfd, 0x34, 0xe5, 0x63, 0xa1, + 0x29, 0xf2, 0x3e, 0xb4, 0xf7, 0xf9, 0xf0, 0x85, 0x38, 0xbe, 0x9a, 0x08, 0xdf, 0xed, 0x3b, 0x5b, + 0x6d, 0x56, 0x31, 0x4a, 0x69, 0x18, 0xbd, 0x14, 0xbe, 0xd7, 0x77, 0xb6, 0xba, 0xac, 0x62, 0x90, + 0x3e, 0x74, 0x8e, 0xa3, 0xb1, 0xf8, 0x3e, 0xe7, 0x89, 0xca, 0xc7, 0x7e, 0x13, 0xad, 0xeb, 0x2c, + 0xf2, 0x10, 0x5a, 0x4f, 0x23, 0x11, 0x8f, 0x32, 0x7f, 0xa5, 0xef, 0x6e, 0x75, 0x76, 0xee, 0x6c, + 0x17, 0x39, 0x6d, 0x23, 0x9f, 0x59, 0x31, 0xa5, 0xb0, 0x16, 0x8c, 0x27, 0x32, 0x55, 0x4c, 0x64, + 0x13, 0x99, 0x64, 0x82, 0xf4, 0xc0, 0x3d, 0x48, 0x53, 0xdf, 0x41, 0xa7, 0xfa, 0x93, 0xfe, 0x0c, + 0xbd, 0xbd, 0x58, 0x0e, 0xcf, 0x06, 0x5c, 0x71, 0x26, 0x7e, 0xca, 0x45, 0xa6, 0xc8, 0x3d, 0x68, + 0x62, 0x65, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x15, 0xfa, 0x0d, 0xc3, 0x45, 0x42, 0x73, 0xd1, 0x1e, + 0xcb, 0xf4, 0x98, 0x21, 0x34, 0x37, 0x8c, 0xa3, 0xa1, 0x29, 0xcf, 0x63, 0x86, 0x20, 0x04, 0xbc, + 0x67, 0x91, 0xb8, 0xb0, 0x35, 0xe1, 0x37, 0x0d, 0x60, 0xbd, 0x16, 0xdf, 0xa6, 0xb9, 0x01, 0x2d, + 0x26, 0x2f, 0x82, 0x41, 0xe6, 0x3b, 0x7d, 0x77, 0xcb, 0x63, 0x96, 0xc2, 0xce, 0xc9, 0x38, 0x1f, + 0x27, 0x5a, 0xd4, 0x40, 0x51, 0xc5, 0xa0, 0xf7, 0xa1, 0x89, 0x6d, 0xd4, 0x55, 0x56, 0xb6, 0xfa, + 0x93, 0xfe, 0xe2, 0x40, 0xfb, 0x90, 0x5f, 0x62, 0x1a, 0x19, 0x79, 0x02, 0xab, 0xa1, 0xe2, 0xc9, + 0x88, 0xa7, 0x23, 0x54, 0xea, 0xec, 0x7c, 0x58, 0xb5, 0xb0, 0x54, 0xdb, 0x2e, 0x74, 0x0e, 0x12, + 0x95, 0x5e, 0xb1, 0xd2, 0xe4, 0xbd, 0x2f, 0xa1, 0x7b, 0x43, 0xa4, 0xe3, 0x9d, 0x89, 0xab, 0xa2, + 0xab, 0x67, 0xe2, 0x4a, 0xd7, 0x7f, 0xce, 0xe3, 0xdc, 0xf4, 0xca, 0x63, 0x86, 0xf8, 0xa2, 0xf1, + 0x99, 0x43, 0x77, 0x81, 0xec, 0xa7, 0x82, 0x2b, 0x81, 0x41, 0x0e, 0x45, 0x96, 0xf1, 0x53, 0xb1, + 0xb8, 0xe3, 0xa6, 0x8b, 0x8d, 0x5a, 0x17, 0xe9, 0x27, 0x40, 0x06, 0x22, 0x16, 0x4a, 0x58, 0xf4, + 0xbd, 0xc2, 0x03, 0x0d, 0x8b, 0x68, 0xb7, 0xeb, 0x92, 0x87, 0xe0, 0x69, 0xf0, 0x62, 0xb0, 0xce, + 0xce, 0xdd, 0xaa, 0x23, 0x25, 0xca, 0x19, 0x2a, 0xd0, 0xa8, 0x70, 0x6a, 0x01, 0x7f, 0x4b, 0x09, + 0x73, 0x40, 0x53, 0x84, 0x72, 0xa7, 0x43, 0x95, 0x2b, 0x64, 0x43, 0xed, 0x16, 0xb5, 0x2e, 0x1b, + 0x8a, 0x9e, 0x96, 0xc9, 0xea, 0x9d, 0x58, 0x26, 0xd9, 0x8f, 0xa0, 0x89, 0xb6, 0x36, 0xdb, 0x99, + 0x6d, 0x33, 0x52, 0xfa, 0xac, 0x4c, 0x75, 0xd9, 0x40, 0xf7, 0xea, 0x81, 0xda, 0x85, 0xdf, 0x1f, + 0xac, 0xae, 0xde, 0x9e, 0x23, 0x6d, 0x63, 0x3c, 0xe1, 0xf7, 0xe2, 0x99, 0x4d, 0x35, 0x52, 0xfb, + 0xd6, 0xeb, 0x96, 0xf9, 0x6e, 0xdf, 0xd5, 0xbe, 0x91, 0xa0, 0x8f, 0xa1, 0x15, 0x0e, 0x5f, 0x88, + 0x31, 0x27, 0x1f, 0xc3, 0x0a, 0xa6, 0x26, 0x32, 0xbb, 0x11, 0x77, 0xa6, 0xe6, 0xcf, 0x0a, 0x39, + 0x1d, 0xd8, 0x92, 0x16, 0x24, 0xd4, 0xc2, 0xd0, 0x99, 0xef, 0xcd, 0xdc, 0x26, 0xcd, 0x67, 0x56, + 0x4c, 0x0f, 0xc0, 0x3d, 0x61, 0x81, 0xde, 0x74, 0xcc, 0xa0, 0xf0, 0x62, 0x29, 0xed, 0xfb, 0x6b, + 0x99, 0x29, 0xdb, 0x20, 0xfc, 0xd6, 0xbc, 0xef, 0x64, 0xaa, 0xb0, 0x3d, 0x5d, 0x86, 0xdf, 0xf4, + 0x47, 0xf0, 0x8e, 0xe4, 0x48, 0x90, 0x35, 0x68, 0x04, 0x03, 0xeb, 0xa3, 0x11, 0x0c, 0xc8, 0x07, + 0xe8, 0xde, 0xf6, 0xa5, 0x5b, 0x25, 0x71, 0xc2, 0x02, 0x86, 0x81, 0x1f, 0x40, 0x37, 0xc8, 0xf6, + 0xa5, 0x4c, 0x47, 0x51, 0xc2, 0x95, 0x4c, 0xd1, 0xeb, 0x2a, 0xbb, 0xc9, 0xa4, 0xbb, 0xd0, 0xd3, + 0xee, 0x43, 0xc5, 0x55, 0x89, 0xbe, 0x0d, 0x68, 0x69, 0x5e, 0x19, 0xce, 0x52, 0xb8, 0xad, 0x5a, + 0xaf, 0x18, 0x2a, 0x12, 0xf4, 0x5b, 0xe3, 0xe1, 0xe0, 0x5c, 0x24, 0xaa, 0x06, 0x0a, 0xa4, 0xd1, + 0x41, 0x97, 0x19, 0x82, 0x50, 0x53, 0x8a, 0xcd, 0x79, 0xad, 0xca, 0x59, 0x73, 0x19, 0xca, 0xe8, + 0x6f, 0x0e, 0x40, 0x91, 0x50, 0x9e, 0x95, 0x26, 0xce, 0x62, 0x13, 0xf2, 0x69, 0xed, 0xf2, 0xcd, + 0xe2, 0xa4, 0x14, 0xb1, 0xda, 0x7d, 0xdc, 0x2a, 0x60, 0x61, 0x21, 0xdf, 0xab, 0xf4, 0x0d, 0xdf, + 0x8e, 0x49, 0x9f, 0x82, 0xee, 0x7e, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0xa1, 0x0d, 0xa3, + 0xec, 0x4f, 0xc5, 0x98, 0xdf, 0x22, 0xf2, 0x00, 0x9a, 0x3a, 0x53, 0x83, 0xcd, 0xd9, 0x32, 0x8c, + 0x90, 0x86, 0x76, 0x3b, 0xe6, 0xc2, 0x8e, 0x80, 0x87, 0x6f, 0xad, 0x85, 0x0b, 0x3e, 0xb3, 0x3d, + 0x70, 0x0f, 0xa3, 0x04, 0x4b, 0x70, 0x99, 0xfe, 0x44, 0x0e, 0xbf, 0xc4, 0x37, 0x49, 0x73, 0xb8, + 0xbe, 0x8f, 0xeb, 0xe6, 0x3a, 0xe8, 0x7d, 0x58, 0x66, 0x67, 0x8b, 0x27, 0xcd, 0xad, 0x3d, 0x69, + 0x21, 0xac, 0x9b, 0x4b, 0xf0, 0x26, 0x9d, 0xfe, 0xd9, 0x80, 0x75, 0x26, 0xb2, 0xe8, 0xa5, 0x08, + 0x92, 0x4c, 0xa5, 0xf9, 0x50, 0x45, 0x32, 0xd1, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xd5, 0x2e, 0x33, + 0xc4, 0xeb, 0x20, 0x89, 0x3c, 0x82, 0xce, 0x34, 0xfa, 0x67, 0x55, 0xeb, 0x2a, 0xe4, 0x11, 0xac, + 0x84, 0x32, 0x4f, 0x87, 0xe5, 0x6e, 0x6f, 0x54, 0xda, 0x26, 0x33, 0x23, 0x66, 0x85, 0x5a, 0x0d, + 0x47, 0xcd, 0x57, 0xe3, 0x88, 0x3c, 0x99, 0xc2, 0x91, 0xdf, 0x42, 0x83, 0x77, 0x2b, 0x83, 0x1b, + 0x62, 0x76, 0x53, 0x9b, 0xfe, 0xea, 0xc0, 0xdb, 0xf5, 0x14, 0x5e, 0x6b, 0x31, 0xca, 0x89, 0x34, + 0xe6, 0x4e, 0xc4, 0x9d, 0x37, 0x11, 0xaf, 0x9a, 0x48, 0xf5, 0x3a, 0x37, 0xeb, 0xaf, 0xf3, 0x19, + 0xdc, 0x9f, 0x19, 0xd3, 0xbe, 0x1c, 0x4f, 0x34, 0x1e, 0xfe, 0xc7, 0xb8, 0xf4, 0xc9, 0x48, 0x53, + 0x3b, 0xa8, 0x36, 0x33, 0x04, 0xfd, 0x1c, 0xde, 0x09, 0x85, 0xaa, 0x0d, 0xa9, 0x40, 0x5b, 0x1f, + 0xdc, 0x23, 0x71, 0xb1, 0xa0, 0x7c, 0x2d, 0xa2, 0x5f, 0x81, 0x7f, 0x32, 0x19, 0x71, 0x25, 0x96, + 0xb2, 0xde, 0x83, 0xd5, 0x63, 0x39, 0x91, 0xb1, 0x3c, 0xbd, 0xba, 0x65, 0xe5, 0x7d, 0x58, 0x31, + 0xf7, 0xd1, 0xfc, 0xb0, 0xb5, 0x59, 0x41, 0xd2, 0xbb, 0x1a, 0xd0, 0x43, 0x1e, 0x0f, 0xf3, 0x58, + 0xa7, 0xa1, 0xff, 0xdc, 0xb2, 0xbd, 0xde, 0x5f, 0xd7, 0x9b, 0xce, 0xdf, 0xd7, 0x9b, 0xce, 0x3f, + 0xd7, 0x9b, 0xce, 0x1f, 0xff, 0x6e, 0xbe, 0xf5, 0xbc, 0x85, 0xff, 0xdd, 0x8f, 0xff, 0x0b, 0x00, + 0x00, 0xff, 0xff, 0xd3, 0x15, 0x68, 0xea, 0x88, 0x0b, 0x00, 0x00, } diff --git a/internal/private.proto b/internal/private.proto index 563b57424..52e587f4b 100644 --- a/internal/private.proto +++ b/internal/private.proto @@ -6,12 +6,10 @@ message IndexMeta { } message FrameMeta { - bool InverseEnabled = 2; string CacheType = 3; uint32 CacheSize = 4; string TimeQuantum = 5; - bool RangeEnabled = 6; - repeated Field Fields = 7; + repeated Field Fields = 7; } message ImportResponse { @@ -37,13 +35,11 @@ message Cache { message MaxSlices { map Standard = 1; - map Inverse = 2; } message CreateSliceMessage { string Index = 1; uint64 Slice = 2; - bool IsInverse = 3; } message DeleteIndexMessage { diff --git a/internal/public.pb.go b/internal/public.pb.go index 22a46565b..0dab2c831 100644 --- a/internal/public.pb.go +++ b/internal/public.pb.go @@ -43,9 +43,9 @@ var _ = math.Inf const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package type Row struct { - Bits []uint64 `protobuf:"varint,1,rep,packed,name=Bits" json:"Bits,omitempty"` - Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` - Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` + Columns []uint64 `protobuf:"varint,1,rep,packed,name=Columns" json:"Columns,omitempty"` + Keys []string `protobuf:"bytes,3,rep,name=Keys" json:"Keys,omitempty"` + Attrs []*Attr `protobuf:"bytes,2,rep,name=Attrs" json:"Attrs,omitempty"` } func (m *Row) Reset() { *m = Row{} } @@ -53,9 +53,9 @@ func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { return fileDescriptorPublic, []int{0} } -func (m *Row) GetBits() []uint64 { +func (m *Row) GetColumns() []uint64 { if m != nil { - return m.Bits + return m.Columns } return nil } @@ -267,12 +267,12 @@ func (m *AttrMap) GetAttrs() []*Attr { } type QueryRequest struct { - Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` - Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` - ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` - Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` - ExcludeAttrs bool `protobuf:"varint,6,opt,name=ExcludeAttrs,proto3" json:"ExcludeAttrs,omitempty"` - ExcludeBits bool `protobuf:"varint,7,opt,name=ExcludeBits,proto3" json:"ExcludeBits,omitempty"` + Query string `protobuf:"bytes,1,opt,name=Query,proto3" json:"Query,omitempty"` + Slices []uint64 `protobuf:"varint,2,rep,packed,name=Slices" json:"Slices,omitempty"` + ColumnAttrs bool `protobuf:"varint,3,opt,name=ColumnAttrs,proto3" json:"ColumnAttrs,omitempty"` + Remote bool `protobuf:"varint,5,opt,name=Remote,proto3" json:"Remote,omitempty"` + ExcludeRowAttrs bool `protobuf:"varint,6,opt,name=ExcludeRowAttrs,proto3" json:"ExcludeRowAttrs,omitempty"` + ExcludeColumns bool `protobuf:"varint,7,opt,name=ExcludeColumns,proto3" json:"ExcludeColumns,omitempty"` } func (m *QueryRequest) Reset() { *m = QueryRequest{} } @@ -308,16 +308,16 @@ func (m *QueryRequest) GetRemote() bool { return false } -func (m *QueryRequest) GetExcludeAttrs() bool { +func (m *QueryRequest) GetExcludeRowAttrs() bool { if m != nil { - return m.ExcludeAttrs + return m.ExcludeRowAttrs } return false } -func (m *QueryRequest) GetExcludeBits() bool { +func (m *QueryRequest) GetExcludeColumns() bool { if m != nil { - return m.ExcludeBits + return m.ExcludeColumns } return false } @@ -575,10 +575,10 @@ func (m *Row) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Bits) > 0 { - dAtA2 := make([]byte, len(m.Bits)*10) + if len(m.Columns) > 0 { + dAtA2 := make([]byte, len(m.Columns)*10) var j1 int - for _, num := range m.Bits { + for _, num := range m.Columns { for num >= 1<<7 { dAtA2[j1] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 @@ -902,20 +902,20 @@ func (m *QueryRequest) MarshalTo(dAtA []byte) (int, error) { } i++ } - if m.ExcludeAttrs { + if m.ExcludeRowAttrs { dAtA[i] = 0x30 i++ - if m.ExcludeAttrs { + if m.ExcludeRowAttrs { dAtA[i] = 1 } else { dAtA[i] = 0 } i++ } - if m.ExcludeBits { + if m.ExcludeColumns { dAtA[i] = 0x38 i++ - if m.ExcludeBits { + if m.ExcludeColumns { dAtA[i] = 1 } else { dAtA[i] = 0 @@ -1263,9 +1263,9 @@ func encodeVarintPublic(dAtA []byte, offset int, v uint64) int { func (m *Row) Size() (n int) { var l int _ = l - if len(m.Bits) > 0 { + if len(m.Columns) > 0 { l = 0 - for _, e := range m.Bits { + for _, e := range m.Columns { l += sovPublic(uint64(e)) } n += 1 + sovPublic(uint64(l)) + l @@ -1405,10 +1405,10 @@ func (m *QueryRequest) Size() (n int) { if m.Remote { n += 2 } - if m.ExcludeAttrs { + if m.ExcludeRowAttrs { n += 2 } - if m.ExcludeBits { + if m.ExcludeColumns { n += 2 } return n @@ -1615,7 +1615,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { break } } - m.Bits = append(m.Bits, v) + m.Columns = append(m.Columns, v) } else if wireType == 2 { var packedLen int for shift := uint(0); ; shift += 7 { @@ -1655,10 +1655,10 @@ func (m *Row) Unmarshal(dAtA []byte) error { break } } - m.Bits = append(m.Bits, v) + m.Columns = append(m.Columns, v) } } else { - return fmt.Errorf("proto: wrong wireType = %d for field Bits", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Columns", wireType) } case 2: if wireType != 2 { @@ -2602,7 +2602,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { m.Remote = bool(v != 0) case 6: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ExcludeAttrs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ExcludeRowAttrs", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -2619,10 +2619,10 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.ExcludeAttrs = bool(v != 0) + m.ExcludeRowAttrs = bool(v != 0) case 7: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ExcludeBits", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ExcludeColumns", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -2639,7 +2639,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { break } } - m.ExcludeBits = bool(v != 0) + m.ExcludeColumns = bool(v != 0) default: iNdEx = preIndex skippy, err := skipPublic(dAtA[iNdEx:]) @@ -3795,50 +3795,50 @@ var ( func init() { proto.RegisterFile("public.proto", fileDescriptorPublic) } var fileDescriptorPublic = []byte{ - // 707 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4a, - 0x14, 0xbe, 0x13, 0x3b, 0x7f, 0x27, 0x49, 0x55, 0x8d, 0xee, 0xed, 0xb5, 0xae, 0xae, 0x82, 0x65, - 0xb1, 0xf0, 0x2a, 0x95, 0xc2, 0x1e, 0x44, 0xfa, 0x23, 0x45, 0x15, 0x15, 0x4c, 0x4a, 0x59, 0xbb, - 0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x08, 0x16, - 0x88, 0x15, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x73, 0xc6, 0x13, 0x3b, 0xad, 0x04, 0x2c, 0xd8, - 0xcd, 0xf7, 0x7d, 0x33, 0xc7, 0xf3, 0xcd, 0xf9, 0x4e, 0x02, 0xc3, 0xac, 0xbc, 0x48, 0xe2, 0xcb, - 0x49, 0x96, 0x2b, 0xad, 0x78, 0x2f, 0x4e, 0xb5, 0xcc, 0xd3, 0x28, 0x09, 0x16, 0xe0, 0x08, 0xb5, - 0xe2, 0x1c, 0xdc, 0x59, 0xac, 0x0b, 0x8f, 0xf9, 0x4e, 0xe8, 0x0a, 0x5a, 0xf3, 0x87, 0xd0, 0x7e, - 0xaa, 0x75, 0x5e, 0x78, 0x2d, 0xdf, 0x09, 0x07, 0xd3, 0x9d, 0x89, 0x3d, 0x34, 0x41, 0x5a, 0x18, - 0x11, 0x4f, 0x9e, 0xc8, 0x75, 0xe1, 0x39, 0xbe, 0x13, 0xf6, 0x05, 0xad, 0x83, 0xc7, 0xe0, 0x3e, - 0x8f, 0xe2, 0x9c, 0xef, 0x40, 0x6b, 0x7e, 0xe8, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd, 0x0f, 0xf9, - 0xdf, 0xd0, 0x3e, 0x50, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x5d, 0x70, 0x4e, 0xe4, 0xda, - 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0xce, 0xa3, 0x64, 0xa3, 0x9e, 0x47, 0x09, - 0x15, 0x71, 0x04, 0x2e, 0xb7, 0xab, 0x38, 0x55, 0x95, 0xe0, 0x25, 0x38, 0xb3, 0x58, 0xa3, 0x28, - 0xd4, 0x6a, 0xf3, 0x55, 0x03, 0xf8, 0x7f, 0xd0, 0x3b, 0x50, 0x49, 0xb9, 0x4c, 0xe7, 0x87, 0xd5, - 0xb7, 0x37, 0x98, 0xff, 0x0f, 0xfd, 0xb3, 0x78, 0x29, 0x0b, 0x1d, 0x2d, 0x33, 0xba, 0x84, 0x23, - 0x6a, 0x22, 0x78, 0x05, 0x23, 0xb3, 0x13, 0xdd, 0x2e, 0xa4, 0xbe, 0xe7, 0xe9, 0xf7, 0x5e, 0xe9, - 0xbe, 0xc7, 0xf7, 0x0c, 0x5c, 0xd4, 0xac, 0xc4, 0x36, 0x12, 0x3e, 0xe9, 0xd9, 0x3a, 0x93, 0xd5, - 0x4d, 0x69, 0xcd, 0x7d, 0x18, 0x2c, 0x74, 0x1e, 0xa7, 0xd7, 0xe7, 0x51, 0x52, 0xca, 0xaa, 0x50, - 0x93, 0x42, 0x8f, 0xf3, 0x54, 0x1b, 0xd9, 0x25, 0x1b, 0x1b, 0x8c, 0x1e, 0x67, 0x4a, 0x25, 0x46, - 0x6c, 0xfb, 0x2c, 0xec, 0x89, 0x9a, 0xe0, 0x63, 0x80, 0xe3, 0x44, 0x45, 0xd5, 0xd9, 0x8e, 0xcf, - 0x42, 0x26, 0x1a, 0x4c, 0xb0, 0x0f, 0x5d, 0xbc, 0xe9, 0xb3, 0x28, 0xab, 0xdd, 0xb2, 0x9f, 0xb8, - 0x0d, 0x3e, 0x30, 0x18, 0xbe, 0x28, 0x65, 0xbe, 0x16, 0xf2, 0x4d, 0x29, 0x0b, 0xea, 0x0a, 0xe1, - 0xca, 0xa5, 0x01, 0x7c, 0x0f, 0x3a, 0x8b, 0x24, 0xbe, 0x94, 0xe6, 0xed, 0x5c, 0x51, 0x21, 0xf4, - 0x5a, 0xbf, 0x79, 0x41, 0x5e, 0x7b, 0xa2, 0x49, 0xe1, 0x49, 0x21, 0x97, 0x4a, 0x5b, 0x33, 0x15, - 0xe2, 0x01, 0x0c, 0x8f, 0x6e, 0x2e, 0x93, 0xf2, 0x4a, 0x9a, 0xa3, 0x1d, 0x52, 0xb7, 0x38, 0xac, - 0x5e, 0x61, 0x4a, 0x7c, 0xd7, 0x54, 0x6f, 0x50, 0xc1, 0x5b, 0x06, 0xa3, 0xea, 0xfa, 0x45, 0xa6, - 0xd2, 0x42, 0x62, 0x8f, 0x8e, 0xf2, 0xdc, 0xf6, 0xe8, 0x28, 0xcf, 0xf9, 0x3e, 0x74, 0x85, 0x2c, - 0xca, 0x44, 0xdb, 0xc6, 0xff, 0x53, 0x3f, 0x85, 0x3d, 0x5b, 0x26, 0x5a, 0xd8, 0x5d, 0xfc, 0x09, - 0xec, 0x6c, 0x05, 0xc9, 0x4c, 0xcc, 0x60, 0xfa, 0x6f, 0x7d, 0x6e, 0x4b, 0x17, 0x77, 0xb6, 0x07, - 0x1f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x03, 0x9a, 0x5c, 0xba, 0xd3, 0x60, 0x3a, 0xaa, 0xab, 0x08, - 0xb5, 0x12, 0x34, 0xd3, 0x43, 0x60, 0xa7, 0x55, 0x86, 0xd8, 0x29, 0x76, 0x0e, 0x67, 0xd2, 0x7e, - 0xb6, 0xd1, 0x39, 0xa4, 0x85, 0x11, 0xb9, 0x07, 0xdd, 0x83, 0xd7, 0x51, 0x7a, 0x2d, 0xaf, 0x28, - 0x43, 0x3d, 0x61, 0x21, 0x9f, 0xd4, 0x33, 0x49, 0x8f, 0x3e, 0x98, 0xf2, 0xba, 0x84, 0x55, 0x44, - 0x3d, 0xb7, 0x36, 0xc4, 0xd8, 0x82, 0x91, 0x09, 0x71, 0xf0, 0x8d, 0xc1, 0x68, 0xbe, 0xcc, 0x54, - 0xae, 0x1b, 0xc1, 0x98, 0xa7, 0x57, 0xf2, 0xc6, 0x06, 0x83, 0x00, 0xb2, 0xc7, 0x79, 0xb4, 0x34, - 0x13, 0xd0, 0x17, 0x06, 0x20, 0x4b, 0x01, 0xa1, 0x40, 0xb8, 0xc2, 0x00, 0x8a, 0x02, 0xce, 0x78, - 0xe1, 0xb9, 0x26, 0x44, 0x06, 0x61, 0xe4, 0xed, 0x88, 0x17, 0x5e, 0x9b, 0xa4, 0x9a, 0xc0, 0xc8, - 0x6f, 0x66, 0x1c, 0x63, 0xe2, 0x84, 0x8e, 0x68, 0x30, 0xf8, 0x0e, 0x42, 0xad, 0xe8, 0x87, 0xad, - 0x4b, 0x3f, 0x6c, 0x16, 0xe2, 0x49, 0x53, 0x86, 0xc4, 0x1e, 0x89, 0x0d, 0x26, 0xf8, 0xc4, 0x80, - 0x1b, 0x8f, 0x34, 0x3c, 0x7f, 0xce, 0x28, 0xee, 0x8d, 0x65, 0x62, 0x1a, 0x83, 0x7b, 0x11, 0xfc, - 0xc2, 0xe6, 0x1e, 0x74, 0xe8, 0x16, 0xd6, 0x62, 0x85, 0xee, 0x98, 0xe8, 0xde, 0x35, 0x31, 0xdb, - 0xfd, 0x7c, 0x3b, 0x66, 0x5f, 0x6e, 0xc7, 0xec, 0xeb, 0xed, 0x98, 0xbd, 0xfb, 0x3e, 0xfe, 0xeb, - 0xa2, 0x43, 0x7f, 0x1c, 0x8f, 0x7e, 0x04, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x35, 0x66, 0x28, 0x48, - 0x06, 0x00, 0x00, + // 709 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x4c, + 0x14, 0xfd, 0x26, 0x76, 0xfe, 0x6e, 0x9a, 0x7c, 0xd5, 0xe8, 0xfb, 0x8a, 0x85, 0x50, 0xb0, 0x2c, + 0x84, 0xbc, 0x4a, 0xa5, 0xb0, 0x07, 0xd1, 0x3f, 0x29, 0xaa, 0xa8, 0x60, 0x5a, 0x8a, 0x58, 0xba, + 0xed, 0xa8, 0x58, 0x72, 0x3c, 0xc6, 0x1e, 0x2b, 0xcd, 0x73, 0xb0, 0xe1, 0x11, 0x78, 0x0c, 0xc4, + 0xaa, 0x4b, 0x1e, 0x01, 0xca, 0x8b, 0xa0, 0x7b, 0xc7, 0x13, 0xbb, 0xa9, 0x04, 0x2c, 0xd8, 0xcd, + 0x39, 0x67, 0xe6, 0x66, 0xce, 0xdc, 0x73, 0x1d, 0xd8, 0xc8, 0xca, 0xb3, 0x24, 0x3e, 0x9f, 0x64, + 0xb9, 0xd2, 0x8a, 0xf7, 0xe2, 0x54, 0xcb, 0x3c, 0x8d, 0x92, 0xe0, 0x2d, 0x38, 0x42, 0x2d, 0xb8, + 0x07, 0xdd, 0x5d, 0x95, 0x94, 0xf3, 0xb4, 0xf0, 0x98, 0xef, 0x84, 0xae, 0xb0, 0x90, 0x3f, 0x82, + 0xf6, 0x73, 0xad, 0xf3, 0xc2, 0x6b, 0xf9, 0x4e, 0x38, 0x98, 0x8e, 0x26, 0xf6, 0xe8, 0x04, 0x69, + 0x61, 0x44, 0xce, 0xc1, 0x3d, 0x94, 0xcb, 0xc2, 0x73, 0x7c, 0x27, 0xec, 0x0b, 0x5a, 0x07, 0x4f, + 0xc1, 0x7d, 0x19, 0xc5, 0x39, 0x1f, 0x41, 0x6b, 0xb6, 0xe7, 0x31, 0x9f, 0x85, 0xae, 0x68, 0xcd, + 0xf6, 0xf8, 0x7f, 0xd0, 0xde, 0x55, 0x65, 0xaa, 0xbd, 0x16, 0x51, 0x06, 0xf0, 0x4d, 0x70, 0x0e, + 0xe5, 0xd2, 0x73, 0x7c, 0x16, 0xf6, 0x05, 0x2e, 0x83, 0x29, 0xf4, 0x4e, 0xa3, 0x64, 0xa5, 0x9e, + 0x46, 0x09, 0x15, 0x71, 0x04, 0x2e, 0x6f, 0x57, 0x71, 0xaa, 0x2a, 0xc1, 0x6b, 0x70, 0x76, 0x62, + 0x8d, 0xa2, 0x50, 0x8b, 0xd5, 0xaf, 0x1a, 0xc0, 0xef, 0x43, 0xcf, 0xb8, 0x9a, 0xed, 0x55, 0xbf, + 0xbd, 0xc2, 0xfc, 0x01, 0xf4, 0x4f, 0xe2, 0xb9, 0x2c, 0x74, 0x34, 0xcf, 0xe8, 0x12, 0x8e, 0xa8, + 0x89, 0xe0, 0x0d, 0x0c, 0xcd, 0x4e, 0x74, 0x7b, 0x2c, 0xf5, 0x1d, 0x4f, 0x7f, 0xf6, 0x4a, 0x77, + 0x3d, 0x7e, 0x62, 0xe0, 0xa2, 0x66, 0x25, 0xb6, 0x92, 0xf0, 0x49, 0x4f, 0x96, 0x99, 0xac, 0x6e, + 0x4a, 0x6b, 0xee, 0xc3, 0xe0, 0x58, 0xe7, 0x71, 0x7a, 0x79, 0x1a, 0x25, 0xa5, 0xac, 0x0a, 0x35, + 0x29, 0xf4, 0x38, 0x4b, 0xb5, 0x91, 0x5d, 0xb2, 0xb1, 0xc2, 0xe8, 0x71, 0x47, 0xa9, 0xc4, 0x88, + 0x6d, 0x9f, 0x85, 0x3d, 0x51, 0x13, 0x7c, 0x0c, 0x70, 0x90, 0xa8, 0xa8, 0x3a, 0xdb, 0xf1, 0x59, + 0xc8, 0x44, 0x83, 0x09, 0xb6, 0xa1, 0x8b, 0x37, 0x7d, 0x11, 0x65, 0xb5, 0x5b, 0xf6, 0x0b, 0xb7, + 0xc1, 0x35, 0x83, 0x8d, 0x57, 0xa5, 0xcc, 0x97, 0x42, 0xbe, 0x2f, 0x65, 0x41, 0x5d, 0x21, 0x5c, + 0xb9, 0x34, 0x80, 0x6f, 0x41, 0xe7, 0x38, 0x89, 0xcf, 0xa5, 0x79, 0x3b, 0x57, 0x54, 0x08, 0xbd, + 0xd6, 0x6f, 0x5e, 0x90, 0xd7, 0x9e, 0x68, 0x52, 0x78, 0x52, 0xc8, 0xb9, 0xd2, 0xd6, 0x4c, 0x85, + 0x78, 0x08, 0xff, 0xee, 0x5f, 0x9d, 0x27, 0xe5, 0x85, 0x14, 0x6a, 0x61, 0x4e, 0x77, 0x68, 0xc3, + 0x3a, 0xcd, 0x1f, 0xc3, 0xa8, 0xa2, 0x6c, 0xfa, 0xbb, 0xb4, 0x71, 0x8d, 0x0d, 0x3e, 0x30, 0x18, + 0x56, 0x56, 0x8a, 0x4c, 0xa5, 0x85, 0xc4, 0x7e, 0xed, 0xe7, 0xb9, 0xed, 0xd7, 0x7e, 0x9e, 0xf3, + 0x6d, 0xe8, 0x0a, 0x59, 0x94, 0x89, 0xb6, 0x21, 0xf8, 0xbf, 0x7e, 0x16, 0x7b, 0xb6, 0x4c, 0xb4, + 0xb0, 0xbb, 0xf8, 0x33, 0x18, 0xdd, 0x0a, 0x95, 0x99, 0x9e, 0xc1, 0xf4, 0x5e, 0x7d, 0xee, 0x96, + 0x2e, 0xd6, 0xb6, 0x07, 0x9f, 0x19, 0x0c, 0x1a, 0x95, 0xf9, 0x43, 0x9a, 0x65, 0xba, 0xd3, 0x60, + 0x3a, 0xac, 0xab, 0x08, 0xb5, 0x10, 0x34, 0xe5, 0x1b, 0xc0, 0x8e, 0xaa, 0x3c, 0xb1, 0x23, 0xec, + 0x22, 0xce, 0xa7, 0xfd, 0xd9, 0x46, 0x17, 0x91, 0x16, 0x46, 0xa4, 0x2f, 0xc3, 0xbb, 0x28, 0xbd, + 0x94, 0x17, 0x94, 0xa7, 0x9e, 0xb0, 0x90, 0x4f, 0xea, 0xf9, 0xa4, 0x06, 0x0c, 0xa6, 0xbc, 0x2e, + 0x61, 0x15, 0x51, 0xcf, 0xb0, 0x0d, 0x34, 0xf6, 0x62, 0x68, 0x02, 0x1d, 0x7c, 0x67, 0x30, 0x9c, + 0xcd, 0x33, 0x95, 0xeb, 0x46, 0x48, 0x66, 0xe9, 0x85, 0xbc, 0xb2, 0x21, 0x21, 0x80, 0xec, 0x41, + 0x1e, 0xcd, 0xcd, 0x34, 0xf4, 0x85, 0x01, 0xc8, 0x52, 0x58, 0x28, 0x1c, 0xae, 0x30, 0x80, 0x62, + 0x81, 0xf3, 0x5e, 0x78, 0xae, 0x09, 0x94, 0x41, 0x18, 0x7f, 0x3b, 0xee, 0x85, 0xd7, 0x26, 0xa9, + 0x26, 0x30, 0xfe, 0xab, 0x79, 0xc7, 0xbc, 0x38, 0xa1, 0x23, 0x1a, 0x0c, 0xbe, 0x83, 0x50, 0x0b, + 0xfa, 0xc8, 0x75, 0xe9, 0x23, 0x67, 0x21, 0x9e, 0x34, 0x65, 0x48, 0xec, 0x91, 0xd8, 0x60, 0x82, + 0x2f, 0x0c, 0xb8, 0xf1, 0x48, 0x83, 0xf4, 0xf7, 0x8c, 0xe2, 0xde, 0x58, 0x26, 0xa6, 0x31, 0xb8, + 0x17, 0xc1, 0x6f, 0x6c, 0x6e, 0x41, 0x87, 0x6e, 0x61, 0x2d, 0x56, 0x68, 0xcd, 0x44, 0x77, 0xdd, + 0xc4, 0xce, 0xe6, 0xf5, 0xcd, 0x98, 0x7d, 0xbd, 0x19, 0xb3, 0x6f, 0x37, 0x63, 0xf6, 0xf1, 0xc7, + 0xf8, 0x9f, 0xb3, 0x0e, 0xfd, 0x95, 0x3c, 0xf9, 0x19, 0x00, 0x00, 0xff, 0xff, 0x03, 0x56, 0xc7, + 0xa4, 0x5a, 0x06, 0x00, 0x00, } diff --git a/internal/public.proto b/internal/public.proto index f2afdb7f9..9207d3a67 100644 --- a/internal/public.proto +++ b/internal/public.proto @@ -3,7 +3,7 @@ syntax = "proto3"; package internal; message Row { - repeated uint64 Bits = 1; + repeated uint64 Columns = 1; repeated string Keys = 3; repeated Attr Attrs = 2; } @@ -49,8 +49,8 @@ message QueryRequest { repeated uint64 Slices = 2; bool ColumnAttrs = 3; bool Remote = 5; - bool ExcludeAttrs = 6; - bool ExcludeBits = 7; + bool ExcludeRowAttrs = 6; + bool ExcludeColumns = 7; } message QueryResponse { diff --git a/pilosa.go b/pilosa.go index bad8badc3..7ddea28eb 100644 --- a/pilosa.go +++ b/pilosa.go @@ -32,10 +32,9 @@ var ( ErrIndexNotFound = errors.New("index not found") // ErrFrameRequired is returned when no frame is specified. - ErrFrameRequired = errors.New("frame required") - ErrFrameExists = errors.New("frame already exists") - ErrFrameNotFound = errors.New("frame not found") - ErrFrameInverseDisabled = errors.New("frame inverse disabled") + ErrFrameRequired = errors.New("frame required") + ErrFrameExists = errors.New("frame already exists") + ErrFrameNotFound = errors.New("frame not found") ErrFieldNotFound = errors.New("field not found") ErrFieldExists = errors.New("field already exists") diff --git a/pql/ast.go b/pql/ast.go index e26432ad3..c3deff9dd 100644 --- a/pql/ast.go +++ b/pql/ast.go @@ -177,34 +177,6 @@ func (c *Call) String() string { return buf.String() } -// SupportsInverse indicates that the call may be on an inverse frame. -func (c *Call) SupportsInverse() bool { - return c.Name == "Bitmap" || c.Name == "TopN" -} - -// IsInverse specifies if the call is for an inverse view. -// Return defaults to false unless absolutely sure of inversion. -func (c *Call) IsInverse(rowLabel, columnLabel string) bool { - if c.SupportsInverse() { - // Top-n has an explicit inverse flag. - if c.Name == "TopN" { - inverse, _ := c.Args["inverse"].(bool) - return inverse - } - - // Bitmap calls use the row/column labels to determine whether inverse. - _, rowOK, rowErr := c.UintArg(rowLabel) - _, columnOK, columnErr := c.UintArg(columnLabel) - if rowErr != nil || columnErr != nil { - return false - } - if !rowOK && columnOK { - return true - } - } - return false -} - // HasConditionArg returns true if any arg is a conditional. func (c *Call) HasConditionArg() bool { for _, v := range c.Args { diff --git a/pql/ast_test.go b/pql/ast_test.go index 9dd3b693f..f5d75e9de 100644 --- a/pql/ast_test.go +++ b/pql/ast_test.go @@ -67,69 +67,3 @@ func TestCondition_Value(t *testing.T) { } }) } - -// Ensure call can be converted into a string. -func TestCall_SupportsInverse(t *testing.T) { - t.Run("Bitmap", func(t *testing.T) { - q, err := pql.ParseString(`Bitmap()`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].SupportsInverse() != true { - t.Fatalf("call should support inverse: %s", q.Calls[0]) - } - }) - t.Run("Count Bitmap", func(t *testing.T) { - q, err := pql.ParseString(`Count(Bitmap())`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].SupportsInverse() == true { - t.Fatalf("call should not support inverse: %s", q.Calls[0]) - } - }) - t.Run("Union Bitmaps", func(t *testing.T) { - q, err := pql.ParseString(`Union(Bitmap(), Bitmap())`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].SupportsInverse() == true { - t.Fatalf("call should not support inverse: %s", q.Calls[0]) - } - }) - -} - -// Ensure call is correctly determined to be against an inverse view. -func TestCall_IsInverse(t *testing.T) { - t.Run("Bitmap Row", func(t *testing.T) { - q, err := pql.ParseString(`Bitmap(frame="f", row=1)`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].IsInverse("row", "col") != false { - t.Fatalf("incorrect call inverse: %s", q.Calls[0]) - } - }) - t.Run("Bitmap Column", func(t *testing.T) { - q, err := pql.ParseString(`Bitmap(frame="f", col=1)`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].IsInverse("row", "col") != true { - t.Fatalf("incorrect call inverse: %s", q.Calls[0]) - } - }) - t.Run("Bitmap Column No Label", func(t *testing.T) { - q, err := pql.ParseString(`Bitmap(frame="f", col=1)`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].IsInverse("rowX", "colX") != false { - t.Fatalf("incorrect call inverse: %s", q.Calls[0]) - } - }) - t.Run("Count", func(t *testing.T) { - q, err := pql.ParseString(`Count(Bitmap(frame="f", col=1))`) - if err != nil { - t.Fatal(err) - } else if q.Calls[0].IsInverse("row", "col") != false { - t.Fatalf("incorrect call inverse: %s", q.Calls[0]) - } - }) - -} diff --git a/roaring/roaring.go b/roaring/roaring.go index cc8466a5f..fafce4d59 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -1461,22 +1461,14 @@ func (c *Container) arrayMax() uint16 { func (c *Container) bitmapMax() uint16 { // Search bitmap in reverse order. - for i := len(c.bitmap) - 1; i >= 0; i-- { + for i := len(c.bitmap); i > 0; i-- { // If value is zero then skip. - v := c.bitmap[i] - if v == 0 { - continue + v := c.bitmap[i-1] + if v != 0 { + r := bits.LeadingZeros64(v) + return uint16((i-1)*64 + 63 - r) } - // Find the highest set bit. - for j := uint16(63); ; j-- { - if v&(1< oldMaxInverseSlices[index] { - oldMaxInverseSlices[index] = newMaxInverse - localIndex.SetRemoteMaxInverseSlice(newMaxInverse) - } - } - return nil } diff --git a/server/cluster_test.go b/server/cluster_test.go index dfb8c9b48..e6777d90c 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -464,12 +464,12 @@ func TestClusterResize_RemoveNode(t *testing.T) { // This is an attempt to ensure there is data on both nodes, but is not guaranteed. // TODO: Deterministic node IDs would ensure consistent results - setBits := "" + setColumns := "" for i := 0; i < 20; i++ { - setBits += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) + setColumns += fmt.Sprintf("SetBit(row=1, frame=\"f\", col=%d) ", i*pilosa.SliceWidth) } - if _, err := m0.Query("i", "", setBits); err != nil { + if _, err := m0.Query("i", "", setColumns); err != nil { t.Fatal(err) } diff --git a/server/server_test.go b/server/server_test.go index b78e8b79f..9092212b2 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -26,7 +26,6 @@ import ( "strings" "testing" "testing/quick" - "time" "github.com/BurntSushi/toml" "github.com/pilosa/pilosa" @@ -69,8 +68,8 @@ func TestMain_Set_Quick(t *testing.T) { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "bits": columnIDs, - "attrs": map[string]interface{}{}, + "columns": columnIDs, + "attrs": map[string]interface{}{}, }, }, }) + "\n" @@ -92,8 +91,8 @@ func TestMain_Set_Quick(t *testing.T) { exp := MustMarshalJSON(map[string]interface{}{ "results": []interface{}{ map[string]interface{}{ - "bits": columnIDs, - "attrs": map[string]interface{}{}, + "columns": columnIDs, + "attrs": map[string]interface{}{}, }, }, }) + "\n" @@ -132,7 +131,7 @@ func TestMain_SetRowAttrs(t *testing.T) { t.Fatal(err) } - // Set bits on different rows in different frames. + // Set columns on different rows in different frames. if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) } else if _, err := m.Query("i", "", `SetBit(row=2, frame="x", col=100)`); err != nil { @@ -157,14 +156,14 @@ func TestMain_SetRowAttrs(t *testing.T) { // Query row x/1. if res, err := m.Query("i", "", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } // Query row x/2. if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -175,19 +174,19 @@ func TestMain_SetRowAttrs(t *testing.T) { // Query rows after reopening. if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":100},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":100},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=3, frame="neg")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":-0.44},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":-0.44},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } // Query row x/2. if res, err := m.Query("i", "", `Bitmap(row=2, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{"x":-200},"bits":[100]}]}`+"\n" { + } else if res != `{"results":[{"attrs":{"x":-200},"columns":[100]}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } } @@ -205,7 +204,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { t.Fatal(err) } - // Set bits on row. + // Set columns on row. if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=100)`); err != nil { t.Fatal(err) } else if _, err := m.Query("i", "", `SetBit(row=1, frame="x", col=101)`); err != nil { @@ -220,7 +219,7 @@ func TestMain_SetColumnAttrs(t *testing.T) { // Query row. if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result: %s", res) } @@ -231,128 +230,11 @@ func TestMain_SetColumnAttrs(t *testing.T) { // Query row after reopening. if res, err := m.Query("i", "columnAttrs=true", `Bitmap(row=1, frame="x")`); err != nil { t.Fatal(err) - } else if res != `{"results":[{"attrs":{},"bits":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { + } else if res != `{"results":[{"attrs":{},"columns":[100,101]}],"columnAttrs":[{"id":100,"attrs":{"foo":"bar"}}]}`+"\n" { t.Fatalf("unexpected result(reopen): %s", res) } } -// Ensure inverse slices get handled correctly in a multi-node query. -func TestMain_InverseSlices(t *testing.T) { - mains := test.MustRunMainWithCluster(t, 2) - - m0 := mains[0] - m1 := mains[1] - - // Make sure to use node0 in the cluster. - var m *test.Main - if m0.Server.NodeID < m1.Server.NodeID { - m = m0 - } else { - m = m1 - } - - // Create frames. - client := m.Client() - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal("create index:", err) - } - if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{InverseEnabled: true}); err != nil { - t.Fatal("create frame:", err) - } - - // Write data on cluster. - if _, err := m.Query("i", "", fmt.Sprintf(` - SetBit(col=1, frame="f", row=1000) - SetBit(col=1, frame="f", row=2000) - SetBit(col=1, frame="f", row=%d) - `, 1*pilosa.SliceWidth)); err != nil { - t.Fatal("setting bits:", err) - } - - time.Sleep(1 * time.Second) - - // Query the cluster. - if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil { - t.Fatal("another bitmap query:", err) - } else if res != fmt.Sprintf(`{"results":[{"attrs":{},"bits":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" { - t.Fatalf("unexpected result: %s", res) - } -} - -// Ensure program can set bits on one cluster and then restore to a second cluster. -func TestMain_FrameRestore(t *testing.T) { - mains1 := test.MustRunMainWithCluster(t, 2) - m10 := mains1[0] - m11 := mains1[1] - - // Create frames. - client := m10.Client() - if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal("create index:", err) - } - if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { - t.Fatal("create frame:", err) - } - - // Write data on first cluster. - if _, err := m10.Query("i", "", ` - SetBit(row=1, frame="f", col=100) - SetBit(row=1, frame="f", col=1000) - SetBit(row=1, frame="f", col=100000) - SetBit(row=1, frame="f", col=200000) - SetBit(row=1, frame="f", col=400000) - SetBit(row=1, frame="f", col=600000) - SetBit(row=1, frame="f", col=800000) - `); err != nil { - t.Fatal("setting bits:", err) - } - - // Query row on first cluster. - if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { - t.Fatal("bitmap query:", err) - } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { - t.Fatalf("unexpected result: %s", res) - } - - // Start second cluster. - mains2 := test.MustRunMainWithCluster(t, 2) - m20 := mains2[0] - defer m20.Close() - m21 := mains2[1] - defer m21.Close() - - // Import from first cluster. - client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil)) - if err != nil { - t.Fatal("new client:", err) - } - client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil)) - if err != nil { - t.Fatal("new client:", err) - } - - if err := m20.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists { - t.Fatal("create new index:", err) - } - if err := m20.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil { - t.Fatal("create new frame:", err) - } - - if err := client20.RestoreFrame(context.Background(), m10.Server.URI.HostPort(), "i", "f"); err != nil { - t.Fatal("restore frame:", err) - } - if err := client21.RestoreFrame(context.Background(), m11.Server.URI.HostPort(), "i", "f"); err != nil { - t.Fatal("restore frame:", err) - } - - // Query row on second cluster. - if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil { - t.Fatal("another bitmap query:", err) - } else if res != `{"results":[{"attrs":{},"bits":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" { - t.Fatalf("2unexpected result: %s", res) - } -} - // Ensure the host can be parsed. func TestConfig_Parse_Host(t *testing.T) { if c, err := ParseConfig(`bind = "local"`); err != nil { @@ -408,7 +290,7 @@ func TestMain_RecalculateHashes(t *testing.T) { t.Fatal("create frame:", err) } - // Set some bits + // Set some columns data := []string{} for rowID := 1; rowID < 10; rowID++ { for columnID := 1; columnID < 100; columnID++ { @@ -416,7 +298,7 @@ func TestMain_RecalculateHashes(t *testing.T) { } } if _, err := cluster[0].Query("i", "", strings.Join(data, "")); err != nil { - t.Fatal("setting bits:", err) + t.Fatal("setting columns:", err) } // Calculate caches on the first node @@ -440,7 +322,7 @@ func TestMain_RecalculateHashes(t *testing.T) { } } -// SetCommand represents a command to set a bit. +// SetCommand represents a command to set a column. type SetCommand struct { ID uint64 Frame string diff --git a/stats_test.go b/stats_test.go index 0ec6a0a32..611b7b2b6 100644 --- a/stats_test.go +++ b/stats_test.go @@ -146,7 +146,7 @@ func TestStatsCount_Bitmap(t *testing.T) { } } -func TestStatsCount_SetBitmapAttrs(t *testing.T) { +func TestStatsCount_SetColumnAttrs(t *testing.T) { hldr := test.MustOpenHolder() defer hldr.Close() @@ -162,8 +162,8 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) { frame.Stats = &MockStats{ mockCount: func(name string, value int64, rate float64) { - if name != "SetBitmapAttrs" { - t.Errorf("Expected SetBitmapAttrs, Results %s", name) + if name != "SetRowAttrs" { + t.Errorf("Expected SetRowAttrs, Results %s", name) } called = true }, diff --git a/test/fragment.go b/test/fragment.go index 39caa8db2..cd1111750 100644 --- a/test/fragment.go +++ b/test/fragment.go @@ -85,7 +85,7 @@ func (f *Fragment) Reopen() error { return nil } -// MustSetBits sets bits on a row. Panic on error. +// MustSetBits sets columns on a row. Panic on error. // This function does not accept a timestamp or quantum. func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { @@ -95,8 +95,8 @@ func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) { } } -// MustClearBits clears bits on a row. Panic on error. -func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) { +// MustClearColumns clears columns on a row. Panic on error. +func (f *Fragment) MustClearColumns(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { if _, err := f.ClearBit(rowID, columnID); err != nil { panic(err) @@ -126,7 +126,7 @@ func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) { s.attrs[id] = m } -// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk. +// GenerateImportFill generates a set of row/col pairs that evenly fill a fragment chunk. func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) { ipct := int(pct * 100) for i := 0; i < SliceWidth*rowN; i++ { diff --git a/view.go b/view.go index e825aa8ef..f90bf64e9 100644 --- a/view.go +++ b/view.go @@ -30,14 +30,13 @@ import ( // View layout modes. const ( ViewStandard = "standard" - ViewInverse = "inverse" ViewFieldPrefix = "field_" ) // IsValidView returns true if name is valid. func IsValidView(name string) bool { - return name == ViewStandard || name == ViewInverse + return name == ViewStandard } // View represents a container for frame data. @@ -252,9 +251,8 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) { // Send the create slice message to all nodes. err := v.broadcaster.SendAsync( &internal.CreateSliceMessage{ - Index: v.index, - Slice: slice, - IsInverse: IsInverseView(v.name), + Index: v.index, + Slice: slice, }) if err != nil { return nil, errors.Wrap(err, "sending message") @@ -428,11 +426,6 @@ func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint6 return r, nil } -// IsInverseView returns true if the view is used for storing an inverted representation. -func IsInverseView(name string) bool { - return strings.HasPrefix(name, ViewInverse) -} - // ViewInfo represents schema information for a view. type ViewInfo struct { Name string `json:"name"` diff --git a/view_test.go b/view_test.go index 7c7f53a26..21834113d 100644 --- a/view_test.go +++ b/view_test.go @@ -72,7 +72,7 @@ func (v *View) Reopen() error { return v.Open() } -// MustSetBits sets bits on a row. Panic on error. +// MustSetBits sets columns on a row. Panic on error. // This function does not accept a timestamp or quantum. func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { @@ -82,7 +82,7 @@ func (v *View) MustSetBits(rowID uint64, columnIDs ...uint64) { } } -// MustClearBits clears bits on a row. Panic on error. +// MustClearColumns clears columns on a row. Panic on error. func (v *View) MustClearBits(rowID uint64, columnIDs ...uint64) { for _, columnID := range columnIDs { if _, err := v.ClearBit(rowID, columnID); err != nil { diff --git a/webui/assets/main.js b/webui/assets/main.js index b5ff2ab3c..265f51997 100644 --- a/webui/assets/main.js +++ b/webui/assets/main.js @@ -581,14 +581,11 @@ function parse_query(query, indexname) { function parse_options(option_str) { var int_keys = ["cacheSize"]; - var bool_keys = ["inverseEnabled"]; var options = {}; for (var i = 0; i < option_str.length; i++) { var parts = option_str[i].split('='); if (int_keys.indexOf(parts[0]) !== -1 ){ options[parts[0]] = Number(parts[1]) - } else if (bool_keys.indexOf(parts[0]) !== -1){ - options[parts[0]] = (parts[1] == "true") } else { options[parts[0]] = parts[1] }