scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 09:05:55 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into optimizeMax

Browse source
This commit is contained in:
tgruben 2018-05-29 10:07:47 -05:00 • committed by GitHub
commit afc8d86161
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
38 changed files with 162 additions and 2125 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.travis.yml
View file

@ -16,10 +16,6 @@ install:
- make install-dep install-statik vendor generate-statik
script:
- make test
# TODO: When we drop support for Go <1.10, we should use `-coverprofile=` on both `go test` and `goveralls` so the test suite doesn't run twice. See https://github.com/pilosa/pilosa/issues/1009
after_success:
- go get github.com/mattn/goveralls
- $HOME/gopath/bin/goveralls -service=travis-ci -ignore "internal/internal.go,internal/public.pb.go,internal/private.pb.go"
before_deploy:
- pip install awscli --user `whoami`
deploy:
@ -33,6 +29,7 @@ deploy:
matrix:
allow_failures:
- go: master
fast_finish: true
notifications:
slack:
secure: "SceWannxoGzeSu9PlEhl6icQFGuTmwax870k20nB2ZGYLjo77UEcwYoFwWvFsdYPa/HCo3JorMTYvMJ15VDJcnKEfzDr+kyXbHWBzUumclIOU/Im3ArEN6waQgyGbbWUQhvJjy4ATaxiOlmCyDV+KhKC9P3+WB33/OQtM3ngjAdTXYHAkfEcpeoOP75um+KsQgbi+hlnqfZdgDa6yIkFjaS3KZEJW1vmcOYYzNsXOA1Ip8j1NY6AjjWZlQorZJ/SYFqdhIv8ST3+a6cQk12u3t6TwZdcr3wmm1qmiW/SaK7UesWlT/YfElIuK8BBq9w1oZHxNKoAmLWTOe7MMisdItmtwgA14eMGl1rvNFlVf9sjsxs4AAzFvSZBZdDfx9XeLCBU5I2WUc/PKUgNQBPMVChxA7gEhtZLndsDdye7LsZASD2yYqjlVlgoZpzRexee/cJgCqUcNKDBHF39ZJYxV4KtZ0prjcSnVmLvuapplzTV4LZ+LyFapCyhiuM/oMJvxgmd7jTtFb5e5EkaHBPN1XwQWZw87yCjKsunTlTe1f1a5qoH/xvJHNpqE/jxOHU3DTLDgTxhb+FwC1Qj9a8bp+UYLw5F4P46ZnHlBGc2O74klv17EqvUMn3JhzASUtyxLGOgJulJ+o83rxJvhSiWt3GQIfkExVPzmz11641ElJI="

80
api.go
View file

@ -431,79 +431,6 @@ func (api *API) FragmentBlocks(ctx context.Context, indexName string, frameName
return blocks, nil
}
// RestoreFrame reads all the data that this host should have for a given frame
// from replicas in the cluster and restores that data to it.
func (api *API) RestoreFrame(ctx context.Context, indexName string, frameName string, host *URI) error {
if err := api.validate(apiRestoreFrame); err != nil {
return errors.Wrap(err, "validating api method")
}
// Create a client for the remote cluster.
client := NewInternalHTTPClientFromURI(host, api.RemoteClient)
// Determine the maximum number of slices.
maxSlices, err := client.MaxSliceByIndex(ctx)
if err != nil {
return errors.Wrap(err, "getting max slice")
}
// Retrieve frame.
f := api.Holder.Frame(indexName, frameName)
if f == nil {
return ErrFrameNotFound
}
// Retrieve list of all views.
views, err := client.FrameViews(ctx, indexName, frameName)
if err != nil {
return errors.Wrap(err, "getting views")
}
// Loop over each slice and import it if this node owns it.
for slice := uint64(0); slice <= maxSlices[indexName]; slice++ {
// Ignore this slice if we don't own it.
if !api.Cluster.OwnsSlice(api.LocalID(), indexName, slice) {
continue
}
// Loop over view names.
for _, view := range views {
// Create view.
v, err := f.CreateViewIfNotExists(view)
if err != nil {
return errors.Wrap(err, "creating view")
}
// Otherwise retrieve the local fragment.
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
return errors.Wrap(err, "creating fragment")
}
// Stream backup from remote node.
rd, err := client.BackupSlice(ctx, indexName, frameName, view, slice)
if err != nil {
return errors.Wrap(err, "getting backup")
} else if rd == nil {
continue // slice doesn't exist
}
// Restore to local frame and always close reader.
if err := func() error {
defer rd.Close()
if _, err := frag.ReadFrom(rd); err != nil {
return errors.Wrap(err, "reading fragment")
}
return nil
}(); err != nil {
return err
}
}
}
return nil
}
// Hosts returns a list of the hosts in the cluster including their ID,
// URL, and which is the coordinator.
func (api *API) Hosts(ctx context.Context) []*Node {
@ -813,12 +740,6 @@ func (api *API) MaxSlices(ctx context.Context) map[string]uint64 {
return api.Holder.MaxSlices()
}
// MaxInverseSlices returns the maximum inverse slice number for each index in a
// map.
func (api *API) MaxInverseSlices(ctx context.Context) map[string]uint64 {
return api.Holder.MaxInverseSlices()
}
// StatsWithTags returns an instance of whatever implementation of StatsClient
// pilosa is using with the given tags.
func (api *API) StatsWithTags(tags []string) StatsClient {
@ -968,7 +889,6 @@ const (
//apiLocalID // not implemented
//apiLongQueryTime // not implemented
apiMarshalFragment
//apiMaxInverseSlices // not implemented
//apiMaxSlices // not implemented
apiQuery
apiRecalculateCaches

204
client.go
View file

@ -15,20 +15,17 @@
package pilosa
import (
"archive/tar"
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"math/rand"
"net/http"
"net/url"
"sort"
"strconv"
"time"
"crypto/tls"
@ -77,16 +74,11 @@ func (c *InternalHTTPClient) Host() *URI { return c.defaultURI }
// MaxSliceByIndex returns the number of slices on a server by index.
func (c *InternalHTTPClient) MaxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByIndex(ctx, false)
}
// MaxInverseSliceByIndex returns the number of inverse slices on a server by index.
func (c *InternalHTTPClient) MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error) {
return c.maxSliceByIndex(ctx, true)
return c.maxSliceByIndex(ctx)
}
// maxSliceByIndex returns the number of slices on a server by index.
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool) (map[string]uint64, error) {
func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context) (map[string]uint64, error) {
// Execute request against the host.
u := uriPathToURL(c.defaultURI, "/slices/max")
@ -112,9 +104,6 @@ func (c *InternalHTTPClient) maxSliceByIndex(ctx context.Context, inverse bool)
return nil, fmt.Errorf("json decode: %s", err)
}
if inverse {
return rsp.Inverse, nil
}
return rsp.Standard, nil
}
@ -524,7 +513,7 @@ func (c *InternalHTTPClient) ExportCSV(ctx context.Context, index, frame, view s
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
} else if !(view == ViewStandard || view == ViewInverse) {
} else if view != ViewStandard {
return ErrInvalidView
}
@ -589,109 +578,6 @@ func (c *InternalHTTPClient) exportNodeCSV(ctx context.Context, node *Node, inde
return nil
}
// BackupTo backs up an entire frame from a cluster to w.
func (c *InternalHTTPClient) BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
// Create tar writer around writer.
tw := tar.NewWriter(w)
// Find the maximum number of slices.
var maxSlices map[string]uint64
var err error
if view == ViewStandard {
maxSlices, err = c.MaxSliceByIndex(ctx)
} else if view == ViewInverse {
maxSlices, err = c.MaxInverseSliceByIndex(ctx)
} else {
return ErrInvalidView
}
if err != nil {
return fmt.Errorf("slice n: %s", err)
}
// Backup every slice to the tar file.
for i := uint64(0); i <= maxSlices[index]; i++ {
if err := c.backupSliceTo(ctx, tw, index, frame, view, i); err != nil {
return errors.Wrap(err, "backing up slice")
}
}
// Close tar file.
if err := tw.Close(); err != nil {
return errors.Wrap(err, "closing")
}
return nil
}
// backupSliceTo backs up a single slice to tw.
func (c *InternalHTTPClient) backupSliceTo(ctx context.Context, tw *tar.Writer, index, frame, view string, slice uint64) error {
// Return error if unable to backup from any slice.
r, err := c.BackupSlice(ctx, index, frame, view, slice)
if err != nil {
return fmt.Errorf("backup slice: slice=%d, err=%s", slice, err)
} else if r == nil {
return nil
}
defer r.Close()
// Read entire buffer to determine file size.
data, err := ioutil.ReadAll(r)
if err != nil {
return errors.Wrap(err, "reading")
} else if err := r.Close(); err != nil {
return errors.Wrap(err, "closing")
}
// Write slice file header.
if err := tw.WriteHeader(&tar.Header{
Name: strconv.FormatUint(slice, 10),
Mode: 0666,
Size: int64(len(data)),
ModTime: time.Now(),
}); err != nil {
return errors.Wrap(err, "writing header")
}
// Write buffer to file.
if _, err := tw.Write(data); err != nil {
return errors.Wrap(err, "writing buffer")
}
return nil
}
// BackupSlice retrieves a streaming backup from a single slice.
// This function tries slice owners until one succeeds.
func (c *InternalHTTPClient) BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error) {
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, index, slice)
if err != nil {
return nil, fmt.Errorf("slice nodes: %s", err)
}
// Try to backup slice from each one until successful.
for _, i := range rand.Perm(len(nodes)) {
r, err := c.backupSliceNode(ctx, index, frame, view, slice, nodes[i])
if err == nil {
return r, nil // successfully attached
} else if err == ErrFragmentNotFound {
return nil, nil // slice doesn't exist
} else if err != nil {
log.Println(err)
continue
}
}
return nil, fmt.Errorf("unable to connect to any owner")
}
func (c *InternalHTTPClient) RetrieveSliceFromURI(ctx context.Context, index, frame, view string, slice uint64, uri URI) (io.ReadCloser, error) {
node := &Node{
URI: uri,
@ -734,86 +620,6 @@ func (c *InternalHTTPClient) backupSliceNode(ctx context.Context, index, frame,
return resp.Body, nil
}
// RestoreFrom restores a frame from a backup file to an entire cluster.
func (c *InternalHTTPClient) RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error {
if index == "" {
return ErrIndexRequired
} else if frame == "" {
return ErrFrameRequired
}
// Create tar reader around input.
tr := tar.NewReader(r)
// Process each file.
for {
hdr, err := tr.Next()
if err == io.EOF {
return nil
} else if err != nil {
return errors.Wrap(err, "opening")
}
// Parse slice from entry name.
slice, err := strconv.ParseUint(hdr.Name, 10, 64)
if err != nil {
return fmt.Errorf("invalid backup entry: %s", hdr.Name)
}
// Read file into buffer.
var buf bytes.Buffer
if _, err := io.CopyN(&buf, tr, hdr.Size); err != nil {
return errors.Wrap(err, "copying")
}
// Restore file to all nodes that own it.
if err := c.restoreSliceFrom(ctx, buf.Bytes(), index, frame, view, slice); err != nil {
return errors.Wrap(err, "restoring")
}
}
}
// restoreSliceFrom restores a single slice to all owning nodes.
func (c *InternalHTTPClient) restoreSliceFrom(ctx context.Context, buf []byte, index, frame, view string, slice uint64) error {
// Retrieve a list of nodes that own the slice.
nodes, err := c.FragmentNodes(ctx, index, slice)
if err != nil {
return fmt.Errorf("slice nodes: %s", err)
}
// Restore slice to each owner.
for _, node := range nodes {
u := nodePathToURL(node, "/fragment/data")
u.RawQuery = url.Values{
"index": {index},
"frame": {frame},
"view": {view},
"slice": {strconv.FormatUint(slice, 10)},
}.Encode()
// Build request.
req, err := http.NewRequest("POST", u.String(), bytes.NewReader(buf))
if err != nil {
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("User-Agent", "pilosa/"+Version)
resp, err := c.HTTPClient.Do(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
resp.Body.Close()
// Return error if response not OK.
if resp.StatusCode != http.StatusOK {
return fmt.Errorf("unexpected status code: host=%s, code=%d", node.URI, resp.StatusCode)
}
}
return nil
}
// CreateFrame creates a new frame on the server.
func (c *InternalHTTPClient) CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error {
if index == "" {
@ -1315,7 +1121,6 @@ func nodePathToURL(node *Node, path string) url.URL {
// I don't want to let it go unquestioned.
type InternalClient interface {
MaxSliceByIndex(ctx context.Context) (map[string]uint64, error)
MaxInverseSliceByIndex(ctx context.Context) (map[string]uint64, error)
Schema(ctx context.Context) ([]*IndexInfo, error)
CreateIndex(ctx context.Context, index string, opt IndexOptions) error
FragmentNodes(ctx context.Context, index string, slice uint64) ([]*Node, error)
@ -1327,9 +1132,6 @@ type InternalClient interface {
EnsureFrame(ctx context.Context, indexName string, frameName string, options FrameOptions) error
ImportValue(ctx context.Context, index, frame, field string, slice uint64, vals []FieldValue) error
ExportCSV(ctx context.Context, index, frame, view string, slice uint64, w io.Writer) error
BackupTo(ctx context.Context, w io.Writer, index, frame, view string) error
BackupSlice(ctx context.Context, index, frame, view string, slice uint64) (io.ReadCloser, error)
RestoreFrom(ctx context.Context, r io.Reader, index, frame, view string) error
CreateFrame(ctx context.Context, index, frame string, opt FrameOptions) error
RestoreFrame(ctx context.Context, host, index, frame string) error
FrameViews(ctx context.Context, index, frame string) ([]string, error)

182
client_test.go
View file

@ -15,7 +15,6 @@
package pilosa_test
import (
"bytes"
"context"
"fmt"
"net/http"
@ -240,60 +239,6 @@ func TestClient_Import(t *testing.T) {
}
}
// Ensure client can bulk import data to an inverse frame.
func TestClient_ImportInverseEnabled(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frameOpts := pilosa.FrameOptions{
InverseEnabled: true,
}
frame, err := idx.CreateFrameIfNotExists("f", frameOpts)
if err != nil {
panic(err)
}
v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse)
if err != nil {
panic(err)
}
f, err := v.CreateFragmentIfNotExists(0)
if err != nil {
panic(err)
}
// Load bitmap into cache to ensure cache gets updated.
f.Row(0)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := test.MustNewClient(s.Host(), defaultClient)
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
{RowID: 200, ColumnID: 5},
{RowID: 200, ColumnID: 6},
}); err != nil {
t.Fatal(err)
}
// Verify data.
if a := f.Row(1).Columns(); !reflect.DeepEqual(a, []uint64{0}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := f.Row(5).Columns(); !reflect.DeepEqual(a, []uint64{0, 200}) {
t.Fatalf("unexpected columns: %+v", a)
}
if a := f.Row(6).Columns(); !reflect.DeepEqual(a, []uint64{200}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
// Ensure client can bulk import value data.
func TestClient_ImportValue(t *testing.T) {
hldr := test.MustOpenHolder()
@ -370,133 +315,6 @@ func TestClient_ImportValue(t *testing.T) {
}
}
// Ensure client backup and restore a frame.
func TestClient_BackupRestore(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(100, SliceWidth, SliceWidth+2)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
// Backup from frame.
var buf bytes.Buffer
if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewStandard); err != nil {
t.Fatal(err)
}
// Restore to a different frame.
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewStandard); err != nil {
t.Fatal(err)
}
// Verify data.
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 1).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{SliceWidth, SliceWidth + 2}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 5).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{(5 * SliceWidth) + 1}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
if a := hldr.Fragment("x", "y", pilosa.ViewStandard, 0).Row(200).Columns(); !reflect.DeepEqual(a, []uint64{20000}) {
t.Fatalf("unexpected columns: %+v", a)
}
}
// Ensure client backup and restore a frame with inverse view.
func TestClient_BackupInverseView(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
frameOpts := pilosa.FrameOptions{
InverseEnabled: true,
}
frame, err := idx.CreateFrameIfNotExists("f", frameOpts)
if err != nil {
panic(err)
}
v, err := frame.CreateViewIfNotExists(pilosa.ViewInverse)
if err != nil {
panic(err)
}
f, err := v.CreateFragmentIfNotExists(0)
if err != nil {
panic(err)
}
f.SetBit(100, 1)
f.SetBit(100, 2)
f.SetBit(100, 3)
f.SetBit(100, SliceWidth-1)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
// Backup from frame.
var buf bytes.Buffer
if err := c.BackupTo(context.Background(), &buf, "i", "f", pilosa.ViewInverse); err != nil {
t.Fatal(err)
}
// Restore to a different frame.
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrameIfNotExists("y", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
}
if err := c.RestoreFrom(context.Background(), &buf, "x", "y", pilosa.ViewInverse); err != nil {
t.Fatal(err)
}
// Verify data.
if a := hldr.Fragment("x", "y", pilosa.ViewInverse, 0).Row(100).Columns(); !reflect.DeepEqual(a, []uint64{1, 2, 3, SliceWidth - 1}) {
t.Fatalf("unexpected columns(0): %+v", a)
}
}
// backup returns error with invalid view
func TestClient_BackupInvalidView(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
c := test.MustNewClient(s.Host(), defaultClient)
// Backup from frame.
var buf bytes.Buffer
err := c.BackupTo(context.Background(), &buf, "i", "f", "invalid_view")
if err != pilosa.ErrInvalidView {
t.Fatal(err)
}
}
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
func TestClient_FragmentBlocks(t *testing.T) {
hldr := test.MustOpenHolder()

13
cluster.go
View file

@ -626,21 +626,14 @@ func (a viewsByFrame) addView(frame, view string) {
func (c *Cluster) fragsByHost(idx *Index) fragsByHost {
// frameViews is a map of frame to slice of views.
frameViews := make(viewsByFrame)
inverseFrameViews := make(viewsByFrame)
for _, frame := range idx.Frames() {
for _, view := range frame.Views() {
if IsInverseView(view.Name()) {
inverseFrameViews.addView(frame.Name(), view.Name())
} else {
frameViews.addView(frame.Name(), view.Name())
}
frameViews.addView(frame.Name(), view.Name())
}
}
std := c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews)
inv := c.fragCombos(idx.Name(), idx.MaxInverseSlice(), inverseFrameViews)
return std.add(inv)
return c.fragCombos(idx.Name(), idx.MaxSlice(), frameViews)
}
// fragCombos returns a map (by uri) of lists of fragments for a given index

2
cluster_test.go
View file

@ -424,8 +424,6 @@ func TestCluster_ResizeStates(t *testing.T) {
// Add Field Data to node0.
if err := tc.CreateFrame("i", "fields", FrameOptions{
InverseEnabled: false,
//CacheType: CacheTypeNone,
Fields: []*Field{
{
Name: "fld0",

56
cmd/backup.go
View file

@ -1,56 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"io"
"os"
"github.com/pilosa/pilosa/ctl"
"github.com/spf13/cobra"
)
var Backuper *ctl.BackupCommand
func NewBackupCmd(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Backuper = ctl.NewBackupCommand(os.Stdin, os.Stdout, os.Stderr)
backupCmd := &cobra.Command{
Use: "backup",
Short: "Backup data from pilosa.",
Long: `
Backs up the view from across the cluster into a single file.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := Backuper.Run(context.Background()); err != nil {
return err
}
return nil
},
}
flags := backupCmd.Flags()
flags.StringVarP(&Backuper.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Backuper.Index, "index", "i", "", "Pilosa index to backup.")
flags.StringVarP(&Backuper.Frame, "frame", "f", "", "Frame to backup.")
flags.StringVarP(&Backuper.View, "view", "v", "", "View to backup.")
flags.StringVarP(&Backuper.Path, "output-file", "o", "", "File to write backup to - default stdout")
ctl.SetTLSConfig(flags, &Backuper.TLS.CertificatePath, &Backuper.TLS.CertificateKeyPath, &Backuper.TLS.SkipVerify)
return backupCmd
}
func init() {
subcommandFns["backup"] = NewBackupCmd
}

53
cmd/backup_test.go
View file

@ -1,53 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/cmd"
)
func TestBackupHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "backup", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "pilosa backup") || err != nil {
t.Fatalf("Command 'backup --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestBackupConfig(t *testing.T) {
tests := []commandTest{
{
args: []string{"backup", "--output-file", "/somefile"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Backuper.Host, "localhost:12345")
v.Check(cmd.Backuper.Index, "myindex")
v.Check(cmd.Backuper.Frame, "f1")
v.Check(cmd.Backuper.Path, "/somefile")
return v.Error()
},
},
}
executeDry(t, tests)
}

1
cmd/export.go
View file

@ -53,7 +53,6 @@ The file does not contain any headers.
flags.StringVarP(&Exporter.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Exporter.Index, "index", "i", "", "Pilosa index to export")
flags.StringVarP(&Exporter.Frame, "frame", "f", "", "Frame to export")
flags.StringVarP(&Exporter.View, "view", "v", "standard", "View to export - default standard")
flags.StringVarP(&Exporter.Path, "output-file", "o", "", "File to write export to - default stdout")
ctl.SetTLSConfig(flags, &Exporter.TLS.CertificatePath, &Exporter.TLS.CertificateKeyPath, &Exporter.TLS.SkipVerify)

8
cmd/export_test.go
View file

@ -44,7 +44,6 @@ frame = "f1"
v.Check(cmd.Exporter.Host, "localhost:12345")
v.Check(cmd.Exporter.Index, "myindex")
v.Check(cmd.Exporter.Frame, "f1")
v.Check(cmd.Exporter.View, "standard")
v.Check(cmd.Exporter.Path, "/somefile")
return v.Error()
},
@ -52,10 +51,3 @@ frame = "f1"
}
executeDry(t, tests)
}
func TestExportInvalidView(t *testing.T) {
output, err := ExecNewRootCommand(t, "export", "-i", "foo", "-f", "bar", "-v", "test")
if !strings.Contains(err.Error(), "invalid view") {
t.Fatalf("Command 'export' with invalid view should error but: err: '%v', output: '%v'", err, output)
}
}

58
cmd/restore.go
View file

@ -1,58 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd
import (
"context"
"io"
"os"
"github.com/spf13/cobra"
"github.com/pilosa/pilosa/ctl"
)
var Restorer *ctl.RestoreCommand
func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
Restorer = ctl.NewRestoreCommand(os.Stdin, os.Stdout, os.Stderr)
restoreCmd := &cobra.Command{
Use: "restore",
Short: "Restore data to pilosa from a backup file.",
Long: `
Restores a view to the cluster from a backup file.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if err := Restorer.Run(context.Background()); err != nil {
return err
}
return nil
},
}
flags := restoreCmd.Flags()
flags.StringVarP(&Restorer.Host, "host", "", "localhost:10101", "host:port of Pilosa.")
flags.StringVarP(&Restorer.Index, "index", "i", "", "Pilosa index to restore into.")
flags.StringVarP(&Restorer.Frame, "frame", "f", "", "Frame to restore into.")
flags.StringVarP(&Restorer.View, "view", "v", "", "View to restore into.")
flags.StringVarP(&Restorer.Path, "input-file", "d", "", "File to restore data from.")
ctl.SetTLSConfig(flags, &Restorer.TLS.CertificatePath, &Restorer.TLS.CertificateKeyPath, &Restorer.TLS.SkipVerify)
return restoreCmd
}
func init() {
subcommandFns["restore"] = NewRestoreCommand
}

53
cmd/restore_test.go
View file

@ -1,53 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cmd_test
import (
"strings"
"testing"
"github.com/pilosa/pilosa/cmd"
)
func TestRestoreHelp(t *testing.T) {
output, err := ExecNewRootCommand(t, "restore", "--help")
if !strings.Contains(output, "Usage:") ||
!strings.Contains(output, "Flags:") ||
!strings.Contains(output, "pilosa restore") || err != nil {
t.Fatalf("Command 'restore --help' not working, err: '%v', output: '%s'", err, output)
}
}
func TestRestoreConfig(t *testing.T) {
tests := []commandTest{
{
args: []string{"restore", "--input-file", "/somefile"},
env: map[string]string{"PILOSA_HOST": "localhost:12345"},
cfgFileContent: `
index = "myindex"
frame = "f1"
`,
validation: func() error {
v := validator{}
v.Check(cmd.Restorer.Host, "localhost:12345")
v.Check(cmd.Restorer.Index, "myindex")
v.Check(cmd.Restorer.Frame, "f1")
v.Check(cmd.Restorer.Path, "/somefile")
return v.Error()
},
},
}
executeDry(t, tests)
}

94
ctl/backup.go
View file

@ -1,94 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"context"
"io"
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// BackupCommand represents a command for backing up a view.
type BackupCommand struct {
// Destination host and port.
Host string
// Name of the index, frame, view to backup.
Index string
Frame string
View string
// Output file to write to.
Path string
// Standard input/output
*pilosa.CmdIO
TLS server.TLSConfig
}
// NewBackupCommand returns a new instance of BackupCommand.
func NewBackupCommand(stdin io.Reader, stdout, stderr io.Writer) *BackupCommand {
return &BackupCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
}
}
// Run executes the backup.
func (cmd *BackupCommand) Run(ctx context.Context) error {
// Validate arguments.
if cmd.Path == "" {
return errors.New("output file required")
}
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return errors.Wrap(err, "creating client")
}
// Open output file.
f, err := os.Create(cmd.Path)
if err != nil {
return errors.Wrap(err, "creating file")
}
defer f.Close()
// Begin streaming backup.
if err := client.BackupTo(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
return errors.Wrap(err, "backing up")
}
// Sync & close file to ensure durability.
if err := f.Sync(); err != nil {
return errors.Wrap(err, "syncing")
} else if err = f.Close(); err != nil {
return errors.Wrap(err, "closing file")
}
return nil
}
func (cmd *BackupCommand) TLSHost() string {
return cmd.Host
}
func (cmd *BackupCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

65
ctl/backup_test.go
View file

@ -1,65 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"bytes"
"context"
"io/ioutil"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
func TestBackupCommand_FileRequired(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewBackupCommand(stdin, stdout, stderr)
err := cm.Run(context.Background())
if err.Error() != "output file required" {
t.Fatalf("expect error: output file required, actual: %s", err)
}
}
func TestBackupCommand_Run(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm := NewBackupCommand(stdin, stdout, stderr)
file, err := ioutil.TempFile("", "import.csv")
cm.Index = "i"
cm.Host = s.Host()
cm.Frame = "f"
cm.View = pilosa.ViewStandard
cm.Path = file.Name()
err = cm.Run(context.Background())
if err != nil {
t.Fatalf("Command not working, error: '%s'", err)
}
}

10
ctl/bench_test.go
View file

@ -15,6 +15,7 @@
package ctl
import (
"bufio"
"bytes"
"context"
"fmt"
@ -86,3 +87,12 @@ func TestBenchCommand_Run(t *testing.T) {
fmt.Println(buf.String())
}
}
// declare stdin, stdout, stderr
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
rder := []byte{}
stdin := bytes.NewReader(rder)
stdout := bufio.NewWriter(&buf)
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}

14
ctl/export.go
View file

@ -33,7 +33,7 @@ type ExportCommand struct {
// Name of the index & frame to export from.
Index string
Frame string
View string
// Filename to export to.
Path string
@ -59,8 +59,6 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
return pilosa.ErrIndexRequired
} else if cmd.Frame == "" {
return pilosa.ErrFrameRequired
} else if !(cmd.View == pilosa.ViewStandard || cmd.View == pilosa.ViewInverse) {
return pilosa.ErrInvalidView
}
// Use output file, if specified.
@ -83,13 +81,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
}
// Determine slice count.
var maxSlices map[string]uint64
if cmd.View == pilosa.ViewStandard {
maxSlices, err = client.MaxSliceByIndex(ctx)
} else if cmd.View == pilosa.ViewInverse {
maxSlices, err = client.MaxInverseSliceByIndex(ctx)
}
maxSlices, err := client.MaxSliceByIndex(ctx)
if err != nil {
return errors.Wrap(err, "getting slice count")
}
@ -97,7 +89,7 @@ func (cmd *ExportCommand) Run(ctx context.Context) error {
// Export each slice.
for slice := uint64(0); slice <= maxSlices[cmd.Index]; slice++ {
logger.Printf("exporting slice: %d", slice)
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, cmd.View, slice, w); err != nil {
if err := client.ExportCSV(ctx, cmd.Index, cmd.Frame, pilosa.ViewStandard, slice, w); err != nil {
return errors.Wrap(err, "exporting")
}
}

8
ctl/export_test.go
View file

@ -41,13 +41,6 @@ func TestExportCommand_Validation(t *testing.T) {
if err != pilosa.ErrFrameRequired {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrFrameRequired, err)
}
cm.Frame = "f"
cm.View = "test"
err = cm.Run(context.Background())
if err != pilosa.ErrInvalidView {
t.Fatalf("Command not working, expect: %s, actual: '%s'", pilosa.ErrInvalidView, err)
}
}
func TestExportCommand_Run(t *testing.T) {
@ -70,7 +63,6 @@ func TestExportCommand_Run(t *testing.T) {
cm.Index = "i"
cm.Frame = "f"
cm.View = pilosa.ViewStandard
if err := cm.Run(context.Background()); err != nil {
t.Fatalf("Export Run doesn't work: %s", err)
}

87
ctl/restore.go
View file

@ -1,87 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"context"
"io"
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/server"
"github.com/pkg/errors"
)
// RestoreCommand represents a command for restoring a frame from a backup.
type RestoreCommand struct {
// Destination host and port.
Host string
// Name of the index & frame to backup.
Index string
Frame string
View string
// Import file to read from.
Path string
// Standard input/output
*pilosa.CmdIO
TLS server.TLSConfig
}
// NewRestoreCommand returns a new instance of RestoreCommand.
func NewRestoreCommand(stdin io.Reader, stdout, stderr io.Writer) *RestoreCommand {
return &RestoreCommand{
CmdIO: pilosa.NewCmdIO(stdin, stdout, stderr),
}
}
// Run executes the restore command.
func (cmd *RestoreCommand) Run(ctx context.Context) error {
// Validate arguments.
if cmd.Path == "" {
return errors.New("backup file required")
}
// Create a client to the server.
client, err := CommandClient(cmd)
if err != nil {
return errors.Wrap(err, "creating client")
}
// Open backup file.
f, err := os.Open(cmd.Path)
if err != nil {
return errors.Wrap(err, "opening file")
}
defer f.Close()
// Restore backup file to the cluster.
if err := client.RestoreFrom(ctx, f, cmd.Index, cmd.Frame, cmd.View); err != nil {
return errors.Wrap(err, "restoring")
}
return nil
}
func (cmd *RestoreCommand) TLSHost() string {
return cmd.Host
}
func (cmd *RestoreCommand) TLSConfiguration() server.TLSConfig {
return cmd.TLS
}

75
ctl/restore_test.go
View file

@ -1,75 +0,0 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ctl
import (
"bufio"
"bytes"
"context"
"io"
"io/ioutil"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
func TestRestoreCommand_FileRequired(t *testing.T) {
cm := RestoreCommand{}
ctx := context.Background()
err := cm.Run(ctx)
if err.Error() != "backup file required" {
t.Fatalf("expect error: output file required, actual: '%s'", err)
}
}
func TestRestoreCommand_Run(t *testing.T) {
file, err := ioutil.TempFile("", "restore.csv")
if err != nil {
t.Fatal(err)
}
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm := NewRestoreCommand(stdin, stdout, stderr)
cm.Path = file.Name()
cm.Index = "i"
cm.Frame = "f"
cm.View = pilosa.ViewStandard
cm.Host = s.Host()
cm.Run(context.Background())
if err != nil {
t.Fatalf("Backup Run doesn't work: %s", err)
}
}
// declare stdin, stdout, stderr
func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
rder := []byte{}
stdin := bytes.NewReader(rder)
stdout := bufio.NewWriter(&buf)
stderr := bufio.NewWriter(&buf)
return stdin, stdout, stderr
}

10
docs/installation.md
View file

@ -49,7 +49,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -58,7 +57,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:
@ -110,7 +108,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -119,7 +116,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:
@ -177,7 +173,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -186,7 +181,6 @@ There are four ways to install Pilosa on MacOS: Use [Homebrew](https://brew.sh/)
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:
@ -268,7 +262,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -277,7 +270,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:
@ -335,7 +327,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
pilosa [command]
Available Commands:
backup Backup data from pilosa.
bench Benchmark operations.
check Do a consistency check on a pilosa data file.
config Print the current configuration.
@ -344,7 +335,6 @@ There are three ways to install Pilosa on Linux: download the binary (recommende
help Help about any command
import Bulk load data into pilosa.
inspect Get stats on a pilosa data file.
restore Restore data to pilosa from a backup file.
server Run Pilosa.
Flags:

143
executor.go
View file

@ -80,36 +80,21 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// Don't bother calculating slices for query types that don't require it.
needsSlices := needsSlices(q.Calls)
// MaxSlice can differ between inverse and standard views, so we need
// to send queries to different slices based on orientation.
var inverseSlices []uint64
// If slices are specified, then use that value for slices or
// inverseSlices. If slices aren't specified, then include all of them.
if len(slices) > 0 {
// For inverse queries, the values of `slices` provided to the Execute() method
// on the remote node actually represents inverseSlices.
inverseSlices = slices
} else if needsSlices {
// If slices are specified, then use that value for slices. If slices aren't
// specified, then include all of them.
if len(slices) == 0 && needsSlices {
// Round up the number of slices.
idx := e.Holder.Index(index)
if idx == nil {
return nil, ErrIndexNotFound
}
maxSlice := idx.MaxSlice()
maxInverseSlice := idx.MaxInverseSlice()
// Generate a slices of all slices.
slices = make([]uint64, maxSlice+1)
for i := range slices {
slices[i] = uint64(i)
}
// Generate a slices of all inverse slices.
inverseSlices = make([]uint64, maxInverseSlice+1)
for i := range inverseSlices {
inverseSlices[i] = uint64(i)
}
}
// Optimize handling for bulk attribute insertion.
@ -120,23 +105,6 @@ func (e *Executor) Execute(ctx context.Context, index string, q *pql.Query, slic
// Execute each call serially.
results := make([]interface{}, 0, len(q.Calls))
for _, call := range q.Calls {
if call.SupportsInverse() && needsSlices {
// Fetch frame & row label based on argument.
frame := call.Args["frame"].(string)
if frame == "" {
frame = DefaultFrame
}
f := e.Holder.Frame(index, frame)
if f == nil {
return nil, ErrFrameNotFound
}
// If this call is to an inverse frame send to a different list of slices.
if call.IsInverse(rowLabel, columnLabel) {
slices = inverseSlices
}
}
v, err := e.executeCall(ctx, index, call, slices, opt)
if err != nil {
return nil, err
@ -580,7 +548,6 @@ func (e *Executor) executeTopNSlices(ctx context.Context, index string, c *pql.C
// executeTopNSlice executes a TopN call for a single slice.
func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Call, slice uint64) ([]Pair, error) {
frame, _ := c.Args["frame"].(string)
inverse, _ := c.Args["inverse"].(bool)
n, _, err := c.UintArg("n")
if err != nil {
return nil, fmt.Errorf("executeTopNSlice: %v", err)
@ -619,9 +586,6 @@ func (e *Executor) executeTopNSlice(ctx context.Context, index string, c *pql.Ca
// Determine view.
view := ViewStandard
if inverse {
view = ViewInverse
}
f := e.Holder.Fragment(index, frame, view, slice)
if f == nil {
@ -685,32 +649,19 @@ func (e *Executor) executeBitmapSlice(ctx context.Context, index string, c *pql.
return nil, ErrFrameNotFound
}
// Return an error if both the row and column label are specified.
rowID, rowOK, rowErr := c.UintArg(rowLabel)
columnID, columnOK, columnErr := c.UintArg(columnLabel)
if rowErr != nil || columnErr != nil {
return nil, fmt.Errorf("Bitmap() error with arg for col: %v or row: %v", columnErr, rowErr)
if rowErr != nil {
return nil, fmt.Errorf("Bitmap() error with arg for row: %v", rowErr)
}
if rowOK && columnOK {
return nil, fmt.Errorf("Bitmap() cannot specify both %s and %s values", rowLabel, columnLabel)
} else if !rowOK && !columnOK {
return nil, fmt.Errorf("Bitmap() must specify either %s or %s values", rowLabel, columnLabel)
if !rowOK {
return nil, fmt.Errorf("Bitmap() must specify %v", rowLabel)
}
// Determine row or column orientation.
view, id := ViewStandard, rowID
if columnOK {
view, id = ViewInverse, columnID
if !f.InverseEnabled() {
return nil, fmt.Errorf("Bitmap() cannot retrieve columns unless inverse storage enabled")
}
}
frag := e.Holder.Fragment(index, frame, view, slice)
frag := e.Holder.Fragment(index, frame, ViewStandard, slice)
if frag == nil {
return NewRow(), nil
}
return frag.Row(id), nil
return frag.Row(rowID), nil
}
// executeIntersectSlice executes a intersect() call for a local slice.
@ -761,26 +712,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
}
// Read row & column id.
columnID, columnOK, err := c.UintArg(columnLabel)
if err != nil {
return nil, fmt.Errorf("executeRangeSlice - reading column: %v", err)
}
rowID, rowOK, err := c.UintArg(rowLabel)
if err != nil {
return nil, fmt.Errorf("executeRangeSlice - reading row: %v", err)
}
// Determine view.
var id uint64
var viewName string
if columnOK && rowOK {
return nil, fmt.Errorf("Range() cannot contain both %q and %q", columnLabel, rowLabel)
} else if !columnOK && !rowOK {
return nil, fmt.Errorf("Range() must specify either %q or %q", columnLabel, rowLabel)
} else if columnOK {
viewName, id = ViewInverse, columnID
} else {
viewName, id = ViewStandard, rowID
if !rowOK {
return nil, fmt.Errorf("Range() must specify %q", rowLabel)
}
// Parse start time.
@ -811,12 +748,12 @@ func (e *Executor) executeRangeSlice(ctx context.Context, index string, c *pql.C
// Union bitmaps across all time-based subframes.
row := &Row{}
for _, view := range ViewsByTimeRange(viewName, startTime, endTime, q) {
for _, view := range ViewsByTimeRange(ViewStandard, startTime, endTime, q) {
f := e.Holder.Fragment(index, frame, view, slice)
if f == nil {
continue
}
row = row.Union(f.Row(id))
row = row.Union(f.Row(rowID))
}
f.Stats.Count("range", 1, 1.0)
return row, nil
@ -1033,7 +970,6 @@ func (e *Executor) executeCount(ctx context.Context, index string, c *pql.Call,
// executeClearBit executes a ClearBit() call.
func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
view, _ := c.Args["view"].(string)
frame, ok := c.Args["frame"].(string)
if !ok {
return false, errors.New("ClearBit() frame required")
@ -1064,31 +1000,7 @@ func (e *Executor) executeClearBit(ctx context.Context, index string, c *pql.Cal
return false, fmt.Errorf("ClearBit col field '%v' required", columnLabel)
}
// Clear bits for each view.
switch view {
case ViewStandard:
return e.executeClearBitView(ctx, index, c, f, view, colID, rowID, opt)
case ViewInverse:
return e.executeClearBitView(ctx, index, c, f, view, rowID, colID, opt)
case "":
var ret bool
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
if f.InverseEnabled() {
if changed, err := e.executeClearBitView(ctx, index, c, f, ViewInverse, rowID, colID, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
}
return ret, nil
default:
return false, fmt.Errorf("invalid view: %s", view)
}
return e.executeClearBitView(ctx, index, c, f, ViewStandard, colID, rowID, opt)
}
// executeClearBitView executes a ClearBit() call for a single view.
@ -1123,7 +1035,6 @@ func (e *Executor) executeClearBitView(ctx context.Context, index string, c *pql
// executeSetBit executes a SetBit() call.
func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call, opt *ExecOptions) (bool, error) {
view, _ := c.Args["view"].(string)
frame, ok := c.Args["frame"].(string)
if !ok {
return false, errors.New("SetBit() field required: frame")
@ -1164,31 +1075,7 @@ func (e *Executor) executeSetBit(ctx context.Context, index string, c *pql.Call,
timestamp = &t
}
// Set bits for each view.
switch view {
case ViewStandard:
return e.executeSetBitView(ctx, index, c, f, view, colID, rowID, timestamp, opt)
case ViewInverse:
return e.executeSetBitView(ctx, index, c, f, view, rowID, colID, timestamp, opt)
case "":
var ret bool
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
if f.InverseEnabled() {
if changed, err := e.executeSetBitView(ctx, index, c, f, ViewInverse, rowID, colID, timestamp, opt); err != nil {
return ret, err
} else if changed {
ret = true
}
}
return ret, nil
default:
return false, fmt.Errorf("invalid view: %s", view)
}
return e.executeSetBitView(ctx, index, c, f, ViewStandard, colID, rowID, timestamp, opt)
}
// executeSetBitView executes a SetBit() call for a specific view.

43
executor_test.go
View file

@ -33,7 +33,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
f, err := index.CreateFrame("f", pilosa.FrameOptions{})
if err != nil {
t.Fatal(err)
}
@ -60,7 +60,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
// Inhibit columns attributes.
// Inhibit column attributes.
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(row=10, frame=f)`), nil, &pilosa.ExecOptions{ExcludeColumns: true}); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{}) {
@ -83,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if _, err := index.CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
@ -100,14 +100,6 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
if err := index.ColumnAttrStore().SetAttrs(SliceWidth+1, map[string]interface{}{"foo": "bar", "baz": uint64(123)}); err != nil {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(col=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{10, 20}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := res[0].(*pilosa.Row).Attrs; !reflect.DeepEqual(attrs, map[string]interface{}{"foo": "bar", "baz": int64(123)}) {
t.Fatalf("unexpected attrs: %s", spew.Sdump(attrs))
}
})
}
@ -412,9 +404,9 @@ func TestExecutor_Execute_TopN(t *testing.T) {
// Set columns for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
} else if _, err := idx.CreateFrame("f", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, row=0, col=0)
@ -431,7 +423,6 @@ func TestExecutor_Execute_TopN(t *testing.T) {
}
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewInverse, 0).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).RecalculateCache()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
@ -445,17 +436,6 @@ func TestExecutor_Execute_TopN(t *testing.T) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
t.Run("Inverse", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{ID: SliceWidth, Count: 3},
{ID: 0, Count: 2},
}) {
t.Fatalf("unexpected result: %s", spew.Sdump(result))
}
})
}
func TestExecutor_Execute_TopN_fill(t *testing.T) {
@ -758,8 +738,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Create frame.
if _, err := index.CreateFrameIfNotExists("f", pilosa.FrameOptions{
InverseEnabled: true,
TimeQuantum: pilosa.TimeQuantum("YMDH"),
TimeQuantum: pilosa.TimeQuantum("YMDH"),
}); err != nil {
t.Fatal(err)
}
@ -788,14 +767,6 @@ func TestExecutor_Execute_Range(t *testing.T) {
}
})
t.Run("Inverse", func(t *testing.T) {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(col=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if columns := res[0].(*pilosa.Row).Columns(); !reflect.DeepEqual(columns, []uint64{1, 10}) {
t.Fatalf("unexpected columns: %+v", columns)
}
})
}
// Ensure a Range(field) query can be executed.
@ -1246,7 +1217,7 @@ func TestExectutor_SetColumnAttrs_ExcludeFrame(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
index.CreateFrame("f", pilosa.FrameOptions{})
targetAttrs := map[string]interface{}{
"foo": "bar",
}

109
frame.go
View file

@ -31,8 +31,7 @@ import (
// Default frame settings.
const (
DefaultCacheType = CacheTypeRanked
DefaultInverseEnabled = false
DefaultCacheType = CacheTypeRanked
// Default ranked frame cache
DefaultCacheSize = 50000
@ -54,11 +53,10 @@ type Frame struct {
Stats StatsClient
// Frame options.
inverseEnabled bool
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
fields []*Field
cacheType string
cacheSize uint32
timeQuantum TimeQuantum
fields []*Field
Logger Logger
}
@ -82,9 +80,8 @@ func NewFrame(path, index, name string) (*Frame, error) {
broadcaster: NopBroadcaster,
Stats: NopStatsClient,
inverseEnabled: DefaultInverseEnabled,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
cacheType: DefaultCacheType,
cacheSize: DefaultCacheSize,
//timeQuantum
//fields
@ -111,37 +108,18 @@ func (f *Frame) MaxSlice() uint64 {
var max uint64
for _, view := range f.views {
if view.name == ViewInverse {
continue
} else if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max {
if viewMaxSlice := view.MaxSlice(); viewMaxSlice > max {
max = viewMaxSlice
}
}
return max
}
// MaxInverseSlice returns the max inverse slice in the frame.
func (f *Frame) MaxInverseSlice() uint64 {
f.mu.RLock()
defer f.mu.RUnlock()
view := f.views[ViewInverse]
if view == nil {
return 0
}
return view.MaxSlice()
}
// CacheType returns the caching mode for the frame.
func (f *Frame) CacheType() string {
return f.cacheType
}
// InverseEnabled returns true if an inverse view is available.
func (f *Frame) InverseEnabled() bool {
return f.inverseEnabled
}
// SetCacheSize sets the cache size for ranked fames. Persists to meta file on update.
// defaults to DefaultCacheSize 50000
func (f *Frame) SetCacheSize(v uint32) error {
@ -179,11 +157,10 @@ func (f *Frame) Options() FrameOptions {
func (f *Frame) options() FrameOptions {
return FrameOptions{
InverseEnabled: f.inverseEnabled,
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
Fields: f.fields,
CacheType: f.cacheType,
CacheSize: f.cacheSize,
TimeQuantum: f.timeQuantum,
Fields: f.fields,
}
}
@ -255,7 +232,6 @@ func (f *Frame) loadMeta() error {
// Read data from meta file.
buf, err := ioutil.ReadFile(filepath.Join(f.path, ".meta"))
if os.IsNotExist(err) {
f.inverseEnabled = DefaultInverseEnabled
f.cacheType = DefaultCacheType
f.cacheSize = DefaultCacheSize
f.timeQuantum = ""
@ -270,7 +246,6 @@ func (f *Frame) loadMeta() error {
}
// Copy metadata fields.
f.inverseEnabled = pb.InverseEnabled
f.cacheType = pb.CacheType
if f.cacheType == "" {
f.cacheType = DefaultCacheType
@ -532,11 +507,6 @@ func (f *Frame) CreateViewIfNotExists(name string) (*View, error) {
// createViewIfNotExistsBase returns the named view, creating it if necessary.
// The returned bool indicates whether the view was created or not.
func (f *Frame) createViewIfNotExistsBase(name string) (*View, bool, error) {
// Don't create inverse views if they are not enabled.
if !f.InverseEnabled() && IsInverseView(name) {
return nil, false, ErrFrameInverseDisabled
}
f.mu.Lock()
defer f.mu.Unlock()
@ -840,16 +810,14 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
timestamp = timestamps[i]
}
var standard, inverse []string
var standard []string
if timestamp == nil {
standard = []string{ViewStandard}
inverse = []string{ViewInverse}
} else {
standard = ViewsByTime(ViewStandard, *timestamp, q)
// In order to match the logic of `SetBit()`, we want bits
// with timestamps to write to both time and standard views.
standard = append(standard, ViewStandard)
inverse = ViewsByTime(ViewInverse, *timestamp, q)
}
// Attach bit to each standard view.
@ -860,34 +828,10 @@ func (f *Frame) Import(rowIDs, columnIDs []uint64, timestamps []*time.Time) erro
data.ColumnIDs = append(data.ColumnIDs, columnID)
dataByFragment[key] = data
}
if f.inverseEnabled {
// Attach reversed bits to each inverse view.
for _, name := range inverse {
key := importKey{View: name, Slice: rowID / SliceWidth}
data := dataByFragment[key]
data.RowIDs = append(data.RowIDs, columnID) // reversed
data.ColumnIDs = append(data.ColumnIDs, rowID) // reversed
dataByFragment[key] = data
}
}
}
// Import into each fragment.
for key, data := range dataByFragment {
// Skip inverse data if inverse is not enabled.
if !f.inverseEnabled && IsInverseView(key.View) {
continue
}
// Re-sort data for inverse views.
if IsInverseView(key.View) {
sort.Sort(importBitSet{
rowIDs: data.RowIDs,
columnIDs: data.ColumnIDs,
})
}
view, err := f.CreateViewIfNotExists(key.View)
if err != nil {
return errors.Wrap(err, "creating view")
@ -1003,11 +947,10 @@ func (p frameInfoSlice) Less(i, j int) bool { return p[i].Name < p[j].Name }
// FrameOptions represents options to set when initializing a frame.
type FrameOptions struct {
InverseEnabled bool `json:"inverseEnabled,omitempty"`
CacheType string `json:"cacheType,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Fields []*Field `json:"fields,omitempty"`
CacheType string `json:"cacheType,omitempty"`
CacheSize uint32 `json:"cacheSize,omitempty"`
TimeQuantum TimeQuantum `json:"timeQuantum,omitempty"`
Fields []*Field `json:"fields,omitempty"`
}
// Encode converts o into its internal representation.
@ -1020,11 +963,10 @@ func encodeFrameOptions(o *FrameOptions) *internal.FrameMeta {
return nil
}
return &internal.FrameMeta{
InverseEnabled: o.InverseEnabled,
CacheType: o.CacheType,
CacheSize: o.CacheSize,
TimeQuantum: string(o.TimeQuantum),
Fields: encodeFields(o.Fields),
CacheType: o.CacheType,
CacheSize: o.CacheSize,
TimeQuantum: string(o.TimeQuantum),
Fields: encodeFields(o.Fields),
}
}
@ -1033,11 +975,10 @@ func decodeFrameOptions(options *internal.FrameMeta) *FrameOptions {
return nil
}
return &FrameOptions{
InverseEnabled: options.InverseEnabled,
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Fields: decodeFields(options.Fields),
CacheType: options.CacheType,
CacheSize: options.CacheSize,
TimeQuantum: TimeQuantum(options.TimeQuantum),
Fields: decodeFields(options.Fields),
}
}

81
handler.go
View file

@ -81,7 +81,7 @@ func NewHandler() *Handler {
func (h *Handler) populateValidators() {
h.validators = map[string]*queryValidationSpec{}
h.validators["GetFragmentNodes"] = queryValidationSpecRequired("slice", "index")
h.validators["GetSliceMax"] = queryValidationSpecRequired().Optional("inverse")
h.validators["GetSliceMax"] = queryValidationSpecRequired()
h.validators["PostQuery"] = queryValidationSpecRequired().Optional("slices", "columnAttrs", "excludeRowAttrs", "excludeColumns")
h.validators["GetExport"] = queryValidationSpecRequired("index", "frame", "view", "slice")
h.validators["GetFragmentData"] = queryValidationSpecRequired("index", "frame", "view", "slice")
@ -119,7 +119,6 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/cluster/resize/set-coordinator", handler.handlePostClusterResizeSetCoordinator).Methods("POST")
router.PathPrefix("/debug/pprof/").Handler(http.DefaultServeMux).Methods("GET")
router.Handle("/debug/vars", expvar.Handler()).Methods("GET")
router.HandleFunc("/fragment/data", handler.handleGetFragmentData).Methods("GET").Name("GetFragmentData")
router.HandleFunc("/schema", handler.handleGetSchema).Methods("GET")
router.HandleFunc("/slices/max", handler.handleGetSlicesMax).Methods("GET") // TODO: deprecate, but it's being used by the client (for backups)
router.HandleFunc("/status", handler.handleGetStatus).Methods("GET")
@ -134,7 +133,6 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/export", handler.handleGetExport).Methods("GET").Name("GetExport")
router.HandleFunc("/fragment/block/data", handler.handleGetFragmentBlockData).Methods("GET")
router.HandleFunc("/fragment/blocks", handler.handleGetFragmentBlocks).Methods("GET").Name("GetFragmentBlocks")
router.HandleFunc("/fragment/data", handler.handlePostFragmentData).Methods("POST").Name("PostFragmentData")
router.HandleFunc("/fragment/nodes", handler.handleGetFragmentNodes).Methods("GET").Name("GetFragmentNodes")
router.HandleFunc("/import", handler.handlePostImport).Methods("POST")
router.HandleFunc("/import-value", handler.handlePostImportValue).Methods("POST")
@ -147,7 +145,6 @@ func NewRouter(handler *Handler) *mux.Router {
router.HandleFunc("/index/{index}/frame/{frame}", handler.handlePostFrame).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}", handler.handleDeleteFrame).Methods("DELETE")
router.HandleFunc("/index/{index}/frame/{frame}/attr/diff", handler.handlePostFrameAttrDiff).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/restore", handler.handlePostFrameRestore).Methods("POST").Name("PostFrameRestore")
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handlePostFrameField).Methods("POST")
router.HandleFunc("/index/{index}/frame/{frame}/fields", handler.handleGetFrameFields).Methods("GET")
router.HandleFunc("/index/{index}/frame/{frame}/field/{field}", handler.handleDeleteFrameField).Methods("DELETE")
@ -303,7 +300,6 @@ func (h *Handler) handlePostQuery(w http.ResponseWriter, r *http.Request) {
func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
if err := json.NewEncoder(w).Encode(getSlicesMaxResponse{
Standard: h.API.MaxSlices(r.Context()),
Inverse: h.API.MaxInverseSlices(r.Context()),
}); err != nil {
h.Logger.Printf("write slices-max response error: %s", err)
}
@ -311,7 +307,6 @@ func (h *Handler) handleGetSlicesMax(w http.ResponseWriter, r *http.Request) {
type getSlicesMaxResponse struct {
Standard map[string]uint64 `json:"standard"`
Inverse map[string]uint64 `json:"inverse"`
}
// handleGetIndexes handles GET /index request.
@ -1017,48 +1012,6 @@ func (h *Handler) handleGetFragmentNodes(w http.ResponseWriter, r *http.Request)
}
}
// handleGetFragmentData handles GET /fragment/data requests.
func (h *Handler) handleGetFragmentData(w http.ResponseWriter, r *http.Request) {
// Read slice parameter.
q := r.URL.Query()
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
if err != nil {
http.Error(w, "slice required", http.StatusBadRequest)
return
}
// Retrieve fragment from holder.
f, err := h.API.MarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice)
if err != nil {
http.Error(w, err.Error(), http.StatusNotFound)
return
}
// Stream fragment to response body.
if _, err := f.WriteTo(w); err != nil {
h.Logger.Printf("fragment backup error: %s", err)
}
}
// handlePostFragmentData handles POST /fragment/data requests.
func (h *Handler) handlePostFragmentData(w http.ResponseWriter, r *http.Request) {
// Read slice parameter.
q := r.URL.Query()
slice, err := strconv.ParseUint(q.Get("slice"), 10, 64)
if err != nil {
http.Error(w, "slice required", http.StatusBadRequest)
return
}
if err = h.API.UnmarshalFragment(r.Context(), q.Get("index"), q.Get("frame"), q.Get("view"), slice, r.Body); err != nil {
if errors.Cause(err) == ErrFrameNotFound {
http.Error(w, ErrFrameNotFound.Error(), http.StatusNotFound)
} else {
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
}
// handleGetFragmentBlockData handles GET /fragment/block/data requests.
func (h *Handler) handleGetFragmentBlockData(w http.ResponseWriter, r *http.Request) {
buf, err := h.API.FragmentBlockData(r.Context(), r.Body)
@ -1111,38 +1064,6 @@ type getFragmentBlocksResponse struct {
Blocks []FragmentBlock `json:"blocks"`
}
// handlePostFrameRestore handles POST /frame/restore requests.
func (h *Handler) handlePostFrameRestore(w http.ResponseWriter, r *http.Request) {
indexName := mux.Vars(r)["index"]
frameName := mux.Vars(r)["frame"]
q := r.URL.Query()
hostStr := q.Get("host")
// Validate query parameters.
if hostStr == "" {
http.Error(w, "host required", http.StatusBadRequest)
return
}
host, err := NewURIFromAddress(hostStr)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
}
err = h.API.RestoreFrame(r.Context(), indexName, frameName, host)
switch errors.Cause(err) {
case nil:
break
case ErrFrameNotFound:
fallthrough
case ErrFragmentNotFound:
http.Error(w, err.Error(), http.StatusNotFound)
default:
http.Error(w, err.Error(), http.StatusInternalServerError)
}
}
// handleGetVersion handles /version requests.
func (h *Handler) handleGetVersion(w http.ResponseWriter, r *http.Request) {
err := json.NewEncoder(w).Encode(struct {

4
handler_internal_test.go
View file

@ -66,8 +66,8 @@ func TestPostFrameRequestUnmarshalJSON(t *testing.T) {
{json: `{"options": 4}`, err: "options is not map[string]interface{}"},
{json: `{"option": {}}`, err: "Unknown key: option:map[]"},
{json: `{"options": {"badKey": "test"}}`, err: "Unknown key: badKey:test"},
{json: `{"options": {"inverseEnabled": true}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true}}},
{json: `{"options": {"inverseEnabled": true, "cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{InverseEnabled: true, CacheType: "type"}}},
{json: `{"options": {"inverseEnabled": true}}`, err: "Unknown key: inverseEnabled:true"},
{json: `{"options": {"cacheType": "type"}}`, expected: postFrameRequest{Options: FrameOptions{CacheType: "type"}}},
{json: `{"options": {"inverse": true, "cacheType": "type"}}`, err: "Unknown key: inverse:true"},
}
for _, test := range tests {

104
handler_test.go
View file

@ -84,12 +84,10 @@ func TestHandler_Schema(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
@ -107,8 +105,8 @@ func TestHandler_Schema(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"inverseEnabled":true,"cacheType":"ranked","cacheSize":50000},"views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -123,12 +121,10 @@ func TestHandler_Status(t *testing.T) {
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
if f, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewStandard, 0, 0, nil); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(pilosa.ViewInverse, 0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFrameIfNotExists("f0", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
@ -209,48 +205,7 @@ func TestHandler_MaxSlices(t *testing.T) {
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0},"inverse":{"i0":0,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can return the maxslice map for the inverse views.
func TestHandler_MaxSlices_Inverse(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true})
if err != nil {
t.Fatal(err)
}
if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+1, nil); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (1*SliceWidth)+2, nil); err != nil {
t.Fatal(err)
} else if _, err := f0.SetBit(pilosa.ViewInverse, 30, (3*SliceWidth)+4, nil); err != nil {
t.Fatal(err)
}
f1, err := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{}).CreateFrame("f1", pilosa.FrameOptions{InverseEnabled: true})
if err != nil {
t.Fatal(err)
}
if _, err := f1.SetBit(pilosa.ViewStandard, 40, (0*SliceWidth)+1, nil); err != nil {
t.Fatal(err)
} else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+2, nil); err != nil {
t.Fatal(err)
} else if _, err := f1.SetBit(pilosa.ViewInverse, 40, (0*SliceWidth)+4, nil); err != nil {
t.Fatal(err)
}
h := test.NewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":0,"i1":0},"inverse":{"i0":3,"i1":0}}`+"\n" {
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
@ -1101,55 +1056,6 @@ type FrameFields struct {
Fields []pilosa.Field
}
// Ensure the handler can backup a fragment and then restore it.
func TestHandler_Fragment_BackupRestore(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set bits in the index.
f0 := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
f0.MustSetBits(100, 1, 2, 3)
// Begin backing up from slice i/f/0.
resp, err := http.Get(s.URL + "/fragment/data?index=i&frame=f&view=standard&slice=0")
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Ensure response came back OK.
if resp.StatusCode != http.StatusOK {
t.Fatalf("unexpected backup status code: %d", resp.StatusCode)
}
// Create frame.
if _, err := hldr.MustCreateIndexIfNotExists("x", pilosa.IndexOptions{}).CreateFrame("y", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
// Restore backup to slice x/y/0.
if resp, err := http.Post(s.URL+"/fragment/data?index=x&frame=y&view=standard&slice=0", "application/octet-stream", resp.Body); err != nil {
t.Fatal(err)
} else if resp.StatusCode != http.StatusOK {
resp.Body.Close()
t.Fatalf("unexpected restore status code: %d", resp.StatusCode)
} else {
resp.Body.Close()
}
// Verify data is correctly restored.
f1 := hldr.Fragment("x", "y", pilosa.ViewStandard, 0)
if f1 == nil {
t.Fatal("fragment x/y/standard/0 not created")
} else if columns := f1.Row(100).Columns(); !reflect.DeepEqual(columns, []uint64{1, 2, 3}) {
t.Fatalf("unexpected restored columns: %+v", columns)
}
}
// Ensure the handler can retrieve the version.
func TestHandler_Version(t *testing.T) {
hldr := test.MustOpenHolder()

10
holder.go
View file

@ -209,15 +209,6 @@ func (h *Holder) MaxSlices() map[string]uint64 {
return a
}
// MaxInverseSlices returns MaxInverseSlice map for all indexes.
func (h *Holder) MaxInverseSlices() map[string]uint64 {
a := make(map[string]uint64)
for _, index := range h.Indexes() {
a[index.Name()] = index.MaxInverseSlice()
}
return a
}
// Schema returns schema information for all indexes, frames, and views.
func (h *Holder) Schema() []*IndexInfo {
var a []*IndexInfo
@ -270,7 +261,6 @@ func (h *Holder) ApplySchema(schema *internal.Schema) error {
func (h *Holder) EncodeMaxSlices() *internal.MaxSlices {
return &internal.MaxSlices{
Standard: h.MaxSlices(),
Inverse: h.MaxInverseSlices(),
}
}

32
index.go
View file

@ -38,8 +38,7 @@ type Index struct {
frames map[string]*Frame
// Max Slice on any node in the cluster, according to this node.
remoteMaxSlice uint64
remoteMaxInverseSlice uint64
remoteMaxSlice uint64
NewAttrStore func(string) AttrStore
@ -64,8 +63,7 @@ func NewIndex(path, name string) (*Index, error) {
name: name,
frames: make(map[string]*Frame),
remoteMaxSlice: 0,
remoteMaxInverseSlice: 0,
remoteMaxSlice: 0,
NewAttrStore: NewNopAttrStore,
columnAttrStore: NopAttrStore,
@ -236,30 +234,6 @@ func (i *Index) SetRemoteMaxSlice(newmax uint64) {
i.remoteMaxSlice = newmax
}
// MaxInverseSlice returns the max inverse slice in the index according to this node.
func (i *Index) MaxInverseSlice() uint64 {
if i == nil {
return 0
}
i.mu.RLock()
defer i.mu.RUnlock()
max := i.remoteMaxInverseSlice
for _, f := range i.frames {
if slice := f.MaxInverseSlice(); slice > max {
max = slice
}
}
return max
}
// SetRemoteMaxInverseSlice sets the remote max inverse slice value received from another node.
func (i *Index) SetRemoteMaxInverseSlice(v uint64) {
i.mu.Lock()
defer i.mu.Unlock()
i.remoteMaxInverseSlice = v
}
// FramePath returns the path to a frame in the index.
func (i *Index) FramePath(name string) string { return filepath.Join(i.path, name) }
@ -359,8 +333,6 @@ func (i *Index) createFrame(name string, opt FrameOptions) (*Frame, error) {
f.cacheSize = opt.CacheSize
}
f.inverseEnabled = opt.InverseEnabled
// Set fields.
f.fields = opt.Fields

42
index_test.go
View file

@ -98,48 +98,6 @@ func TestIndex_CreateFrame(t *testing.T) {
}
})
t.Run("ErrInverseRangeAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()
frame, err := index.CreateFrame("f", pilosa.FrameOptions{
InverseEnabled: true,
Fields: []*pilosa.Field{
&pilosa.Field{
Name: "myfield",
Type: pilosa.FieldTypeInt,
Min: -20,
Max: 100,
},
},
})
if err != nil {
t.Fatal(err)
}
ch, err := frame.SetBit(pilosa.ViewStandard, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetBit(pilosa.ViewInverse, 1, 2, nil)
if !ch || err != nil {
t.Fatal(ch, err)
}
ch, err = frame.SetFieldValue(1, "myfield", 87)
if !ch || err != nil {
t.Fatal(ch, err)
}
views := frame.Views()
if len(views) != 3 {
var names string
for _, v := range views {
names = names + v.Name() + " "
}
t.Fatalf("Unexpected views: %s", names)
}
})
t.Run("ErrRangeCacheAllowed", func(t *testing.T) {
index := test.MustOpenIndex()
defer index.Close()

369
internal/private.pb.go
View file

@ -70,11 +70,10 @@ func (*IndexMeta) ProtoMessage() {}
func (*IndexMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{0} }
type FrameMeta struct {
InverseEnabled bool `protobuf:"varint,2,opt,name=InverseEnabled,proto3" json:"InverseEnabled,omitempty"`
CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"`
TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"`
CacheType string `protobuf:"bytes,3,opt,name=CacheType,proto3" json:"CacheType,omitempty"`
CacheSize uint32 `protobuf:"varint,4,opt,name=CacheSize,proto3" json:"CacheSize,omitempty"`
TimeQuantum string `protobuf:"bytes,5,opt,name=TimeQuantum,proto3" json:"TimeQuantum,omitempty"`
Fields []*Field `protobuf:"bytes,7,rep,name=Fields" json:"Fields,omitempty"`
}
func (m *FrameMeta) Reset() { *m = FrameMeta{} }
@ -82,13 +81,6 @@ func (m *FrameMeta) String() string { return proto.CompactTextString(
func (*FrameMeta) ProtoMessage() {}
func (*FrameMeta) Descriptor() ([]byte, []int) { return fileDescriptorPrivate, []int{1} }
func (m *FrameMeta) GetInverseEnabled() bool {
if m != nil {
return m.InverseEnabled
}
return false
}
func (m *FrameMeta) GetCacheType() string {
if m != nil {
return m.CacheType
@ -223,7 +215,6 @@ func (m *Cache) GetIDs() []uint64 {
type MaxSlices struct {
Standard map[string]uint64 `protobuf:"bytes,1,rep,name=Standard" json:"Standard,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
Inverse map[string]uint64 `protobuf:"bytes,2,rep,name=Inverse" json:"Inverse,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
}
func (m *MaxSlices) Reset() { *m = MaxSlices{} }
@ -238,17 +229,9 @@ func (m *MaxSlices) GetStandard() map[string]uint64 {
return nil
}
func (m *MaxSlices) GetInverse() map[string]uint64 {
if m != nil {
return m.Inverse
}
return nil
}
type CreateSliceMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
IsInverse bool `protobuf:"varint,3,opt,name=IsInverse,proto3" json:"IsInverse,omitempty"`
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
Slice uint64 `protobuf:"varint,2,opt,name=Slice,proto3" json:"Slice,omitempty"`
}
func (m *CreateSliceMessage) Reset() { *m = CreateSliceMessage{} }
@ -270,13 +253,6 @@ func (m *CreateSliceMessage) GetSlice() uint64 {
return 0
}
func (m *CreateSliceMessage) GetIsInverse() bool {
if m != nil {
return m.IsInverse
}
return false
}
type DeleteIndexMessage struct {
Index string `protobuf:"bytes,1,opt,name=Index,proto3" json:"Index,omitempty"`
}
@ -1059,16 +1035,6 @@ func (m *FrameMeta) MarshalTo(dAtA []byte) (int, error) {
_ = i
var l int
_ = l
if m.InverseEnabled {
dAtA[i] = 0x10
i++
if m.InverseEnabled {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
if len(m.CacheType) > 0 {
dAtA[i] = 0x1a
i++
@ -1289,22 +1255,6 @@ func (m *MaxSlices) MarshalTo(dAtA []byte) (int, error) {
i = encodeVarintPrivate(dAtA, i, uint64(v))
}
}
if len(m.Inverse) > 0 {
for k, _ := range m.Inverse {
dAtA[i] = 0x12
i++
v := m.Inverse[k]
mapSize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v))
i = encodeVarintPrivate(dAtA, i, uint64(mapSize))
dAtA[i] = 0xa
i++
i = encodeVarintPrivate(dAtA, i, uint64(len(k)))
i += copy(dAtA[i:], k)
dAtA[i] = 0x10
i++
i = encodeVarintPrivate(dAtA, i, uint64(v))
}
}
return i, nil
}
@ -1334,16 +1284,6 @@ func (m *CreateSliceMessage) MarshalTo(dAtA []byte) (int, error) {
i++
i = encodeVarintPrivate(dAtA, i, uint64(m.Slice))
}
if m.IsInverse {
dAtA[i] = 0x18
i++
if m.IsInverse {
dAtA[i] = 1
} else {
dAtA[i] = 0
}
i++
}
return i, nil
}
@ -2306,9 +2246,6 @@ func (m *IndexMeta) Size() (n int) {
func (m *FrameMeta) Size() (n int) {
var l int
_ = l
if m.InverseEnabled {
n += 2
}
l = len(m.CacheType)
if l > 0 {
n += 1 + l + sovPrivate(uint64(l))
@ -2407,14 +2344,6 @@ func (m *MaxSlices) Size() (n int) {
n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize))
}
}
if len(m.Inverse) > 0 {
for k, v := range m.Inverse {
_ = k
_ = v
mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + 1 + sovPrivate(uint64(v))
n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize))
}
}
return n
}
@ -2428,9 +2357,6 @@ func (m *CreateSliceMessage) Size() (n int) {
if m.Slice != 0 {
n += 1 + sovPrivate(uint64(m.Slice))
}
if m.IsInverse {
n += 2
}
return n
}
@ -2936,26 +2862,6 @@ func (m *FrameMeta) Unmarshal(dAtA []byte) error {
return fmt.Errorf("proto: FrameMeta: illegal tag %d (wire type %d)", fieldNum, wire)
}
switch fieldNum {
case 2:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field InverseEnabled", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.InverseEnabled = bool(v != 0)
case 3:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field CacheType", wireType)
@ -3761,113 +3667,6 @@ func (m *MaxSlices) Unmarshal(dAtA []byte) error {
}
m.Standard[mapkey] = mapvalue
iNdEx = postIndex
case 2:
if wireType != 2 {
return fmt.Errorf("proto: wrong wireType = %d for field Inverse", wireType)
}
var msglen int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
msglen |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
if msglen < 0 {
return ErrInvalidLengthPrivate
}
postIndex := iNdEx + msglen
if postIndex > l {
return io.ErrUnexpectedEOF
}
if m.Inverse == nil {
m.Inverse = make(map[string]uint64)
}
var mapkey string
var mapvalue uint64
for iNdEx < postIndex {
entryPreIndex := iNdEx
var wire uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
wire |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
fieldNum := int32(wire >> 3)
if fieldNum == 1 {
var stringLenmapkey uint64
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
stringLenmapkey |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
intStringLenmapkey := int(stringLenmapkey)
if intStringLenmapkey < 0 {
return ErrInvalidLengthPrivate
}
postStringIndexmapkey := iNdEx + intStringLenmapkey
if postStringIndexmapkey > l {
return io.ErrUnexpectedEOF
}
mapkey = string(dAtA[iNdEx:postStringIndexmapkey])
iNdEx = postStringIndexmapkey
} else if fieldNum == 2 {
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
mapvalue |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
} else {
iNdEx = entryPreIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPrivate
}
if (iNdEx + skippy) > postIndex {
return io.ErrUnexpectedEOF
}
iNdEx += skippy
}
}
m.Inverse[mapkey] = mapvalue
iNdEx = postIndex
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -3966,26 +3765,6 @@ func (m *CreateSliceMessage) Unmarshal(dAtA []byte) error {
break
}
}
case 3:
if wireType != 0 {
return fmt.Errorf("proto: wrong wireType = %d for field IsInverse", wireType)
}
var v int
for shift := uint(0); ; shift += 7 {
if shift >= 64 {
return ErrIntOverflowPrivate
}
if iNdEx >= l {
return io.ErrUnexpectedEOF
}
b := dAtA[iNdEx]
iNdEx++
v |= (int(b) & 0x7F) << shift
if b < 0x80 {
break
}
}
m.IsInverse = bool(v != 0)
default:
iNdEx = preIndex
skippy, err := skipPrivate(dAtA[iNdEx:])
@ -7218,74 +6997,70 @@ var (
func init() { proto.RegisterFile("private.proto", fileDescriptorPrivate) }
var fileDescriptorPrivate = []byte{
// 1099 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x4f, 0x6f, 0x1b, 0x45,
0x14, 0x67, 0xbd, 0x6b, 0x27, 0x7e, 0xae, 0x53, 0x67, 0x5a, 0xca, 0xb6, 0xaa, 0x82, 0x19, 0x15,
0x6a, 0x38, 0x44, 0x25, 0xbd, 0x40, 0xa1, 0x52, 0x95, 0x38, 0x15, 0x8b, 0x48, 0x04, 0xe3, 0xa4,
0x07, 0x24, 0x90, 0x26, 0xf6, 0x28, 0x5d, 0x65, 0xbd, 0x6b, 0x76, 0xc7, 0xf9, 0xd3, 0x03, 0x67,
0x2e, 0xdc, 0x11, 0x1f, 0x85, 0x4f, 0xc0, 0x91, 0x8f, 0x80, 0xc2, 0x07, 0x01, 0xbd, 0x37, 0xb3,
0x7f, 0x62, 0x3b, 0x4d, 0x09, 0xdc, 0xe6, 0xfd, 0x9d, 0xdf, 0xfb, 0x3b, 0xbb, 0xd0, 0x9e, 0xa4,
0xe1, 0xb1, 0xd4, 0x6a, 0x7d, 0x92, 0x26, 0x3a, 0x61, 0xcb, 0x61, 0xac, 0x55, 0x1a, 0xcb, 0x88,
0xb7, 0xa0, 0x19, 0xc4, 0x23, 0x75, 0xba, 0xa3, 0xb4, 0xe4, 0xbf, 0x39, 0xd0, 0x7c, 0x9e, 0xca,
0xb1, 0x42, 0x8a, 0x7d, 0x00, 0x2b, 0x41, 0x7c, 0xac, 0xd2, 0x4c, 0x6d, 0xc7, 0xf2, 0x20, 0x52,
0x23, 0xbf, 0xd6, 0x75, 0x7a, 0xcb, 0x62, 0x86, 0xcb, 0xee, 0x43, 0x73, 0x4b, 0x0e, 0x5f, 0xaa,
0xbd, 0xb3, 0x89, 0xf2, 0xdd, 0xae, 0xd3, 0x6b, 0x8a, 0x92, 0x51, 0x48, 0x07, 0xe1, 0x2b, 0xe5,
0x7b, 0x5d, 0xa7, 0xd7, 0x16, 0x25, 0x83, 0x75, 0xa1, 0xb5, 0x17, 0x8e, 0xd5, 0x37, 0x53, 0x19,
0xeb, 0xe9, 0xd8, 0xaf, 0x93, 0x75, 0x95, 0xc5, 0x1e, 0x42, 0xe3, 0x79, 0xa8, 0xa2, 0x51, 0xe6,
0x2f, 0x75, 0xdd, 0x5e, 0x6b, 0xe3, 0xe6, 0x7a, 0x8e, 0x7d, 0x9d, 0xf8, 0xc2, 0x8a, 0x39, 0x87,
0x95, 0x60, 0x3c, 0x49, 0x52, 0x2d, 0x54, 0x36, 0x49, 0xe2, 0x4c, 0xb1, 0x0e, 0xb8, 0xdb, 0x69,
0xea, 0x3b, 0xe4, 0x14, 0x8f, 0xfc, 0x47, 0xe8, 0x6c, 0x46, 0xc9, 0xf0, 0xa8, 0x2f, 0xb5, 0x14,
0xea, 0x87, 0xa9, 0xca, 0x34, 0xbb, 0x0d, 0x75, 0xca, 0x80, 0xd5, 0x33, 0x04, 0x72, 0x29, 0x13,
0x14, 0x73, 0x53, 0x18, 0x02, 0xb9, 0x64, 0x4f, 0x61, 0x7a, 0xc2, 0x10, 0xc8, 0x1d, 0x44, 0xe1,
0xd0, 0x84, 0xe7, 0x09, 0x43, 0x30, 0x06, 0xde, 0x8b, 0x50, 0x9d, 0xd8, 0x98, 0xe8, 0xcc, 0x03,
0x58, 0xad, 0xdc, 0x6f, 0x61, 0xde, 0x81, 0x86, 0x48, 0x4e, 0x82, 0x7e, 0xe6, 0x3b, 0x5d, 0xb7,
0xe7, 0x09, 0x4b, 0x51, 0xe6, 0x92, 0x68, 0x3a, 0x8e, 0x51, 0x54, 0x23, 0x51, 0xc9, 0xe0, 0x77,
0xa1, 0x4e, 0x69, 0xc4, 0x28, 0x4b, 0x5b, 0x3c, 0xf2, 0xbf, 0x1d, 0x68, 0xee, 0xc8, 0x53, 0x82,
0x91, 0xb1, 0xa7, 0xb0, 0x3c, 0xd0, 0x32, 0x1e, 0xc9, 0x74, 0x44, 0x4a, 0xad, 0x8d, 0xf7, 0xca,
0x14, 0x16, 0x6a, 0xeb, 0xb9, 0xce, 0x76, 0xac, 0xd3, 0x33, 0x51, 0x98, 0xb0, 0x27, 0xb0, 0x64,
0xeb, 0x4d, 0x18, 0x5a, 0x1b, 0xdd, 0x45, 0xd6, 0x45, 0x4b, 0xa0, 0x71, 0x6e, 0x70, 0xef, 0x33,
0x68, 0x5f, 0x70, 0x8b, 0x58, 0x8f, 0xd4, 0x59, 0x5e, 0x91, 0x23, 0x75, 0x86, 0xb9, 0x3b, 0x96,
0xd1, 0xd4, 0xe4, 0xd9, 0x13, 0x86, 0x78, 0x52, 0xfb, 0xc4, 0xb9, 0xf7, 0x04, 0x6e, 0x54, 0xbd,
0xfe, 0x1b, 0x5b, 0xfe, 0x3d, 0xb0, 0xad, 0x54, 0x49, 0xad, 0x08, 0xde, 0x8e, 0xca, 0x32, 0x79,
0xa8, 0x2e, 0xaf, 0xb4, 0xa9, 0x5e, 0xad, 0x5a, 0xbd, 0xfb, 0xd0, 0x0c, 0xb2, 0x3c, 0x70, 0x97,
0xfa, 0xbe, 0x64, 0xf0, 0x8f, 0x80, 0xf5, 0x55, 0xa4, 0xb4, 0xb2, 0xb3, 0xf3, 0x1a, 0xff, 0x7c,
0x90, 0x63, 0xb9, 0x5a, 0x97, 0x3d, 0x04, 0x0f, 0x47, 0x8f, 0xa0, 0xb4, 0x36, 0x6e, 0x95, 0x99,
0x2e, 0x66, 0x54, 0x90, 0x02, 0x0f, 0x73, 0xa7, 0x76, 0x5c, 0xaf, 0x08, 0x70, 0x41, 0x2b, 0xe7,
0x57, 0xb9, 0xb3, 0x57, 0x15, 0x0b, 0xc0, 0x5e, 0xf5, 0x2c, 0x8f, 0xf5, 0xba, 0x57, 0xf1, 0xc3,
0x02, 0x2c, 0x4e, 0xea, 0x75, 0xc0, 0xbe, 0x0f, 0x75, 0xb2, 0xb5, 0x68, 0xe7, 0x76, 0x80, 0x91,
0xf2, 0x17, 0x05, 0xd4, 0xeb, 0x5e, 0x74, 0xbb, 0x7a, 0x51, 0x33, 0xf7, 0xfb, 0xad, 0xd5, 0xc5,
0x99, 0xde, 0x45, 0x1b, 0xe3, 0x89, 0xce, 0x97, 0xd7, 0x6c, 0x26, 0x91, 0xe8, 0x1b, 0x97, 0x40,
0xe6, 0xbb, 0x5d, 0x17, 0x7d, 0x13, 0xc1, 0x1f, 0x43, 0x63, 0x30, 0x7c, 0xa9, 0xc6, 0x92, 0x7d,
0x88, 0x93, 0x36, 0x52, 0xa7, 0x2a, 0xb3, 0x73, 0x7a, 0x73, 0xa6, 0xfe, 0x22, 0x97, 0xf3, 0xbe,
0x0d, 0xe9, 0x12, 0x40, 0x0d, 0xba, 0x3a, 0xf3, 0xbd, 0xb9, 0x8d, 0x89, 0x7c, 0x61, 0xc5, 0x7c,
0x1b, 0xdc, 0x7d, 0x11, 0xe0, 0xfe, 0x21, 0x04, 0xb9, 0x17, 0x4b, 0xa1, 0xef, 0x2f, 0x92, 0x4c,
0xdb, 0x04, 0xd1, 0x19, 0x79, 0x5f, 0x27, 0xa9, 0xa6, 0xf4, 0xb4, 0x05, 0x9d, 0xf9, 0x77, 0xe0,
0xed, 0x26, 0x23, 0xc5, 0x56, 0xa0, 0x16, 0xf4, 0xad, 0x8f, 0x5a, 0xd0, 0x67, 0xef, 0x92, 0x7b,
0x9b, 0x97, 0x76, 0x09, 0x62, 0x5f, 0x04, 0x82, 0x2e, 0x7e, 0x00, 0xed, 0x20, 0xdb, 0x4a, 0x92,
0x74, 0x14, 0xc6, 0x52, 0x27, 0xa9, 0x9d, 0xb3, 0x8b, 0x4c, 0xfe, 0x0c, 0x3a, 0xe8, 0x7e, 0xa0,
0xa5, 0x2e, 0xba, 0xef, 0x0e, 0x34, 0x90, 0x57, 0x5c, 0x67, 0x29, 0x9a, 0x65, 0xd4, 0xcb, 0x8b,
0x4a, 0x04, 0xff, 0xca, 0x78, 0xd8, 0x3e, 0x56, 0xb1, 0xae, 0x34, 0x05, 0xd1, 0xe4, 0xa0, 0x2d,
0x0c, 0xc1, 0xb8, 0x09, 0xc5, 0x62, 0x5e, 0x29, 0x31, 0x23, 0x57, 0x90, 0x8c, 0xff, 0xec, 0x00,
0xe4, 0x80, 0xa6, 0x59, 0x61, 0xe2, 0x5c, 0x6e, 0xc2, 0x3e, 0xae, 0xec, 0xe3, 0xf9, 0x3e, 0x29,
0x44, 0xa2, 0xb2, 0xb5, 0x7b, 0x79, 0x5b, 0xd8, 0x96, 0xef, 0x94, 0xfa, 0x86, 0x6f, 0xcb, 0x84,
0xab, 0xa0, 0xbd, 0x15, 0x4d, 0x33, 0xad, 0x52, 0x8b, 0x08, 0xdf, 0x0d, 0xc3, 0x28, 0xf2, 0x53,
0x32, 0x16, 0xa7, 0x88, 0x3d, 0x80, 0x3a, 0x22, 0x35, 0xbd, 0x39, 0x1f, 0x86, 0x11, 0xf2, 0x81,
0x9d, 0x8e, 0x85, 0x6d, 0xc7, 0xc0, 0xa3, 0x2f, 0x00, 0xdb, 0x2e, 0xf4, 0xf8, 0x77, 0xc0, 0xdd,
0x09, 0x63, 0x0a, 0xc1, 0x15, 0x78, 0x24, 0x8e, 0x3c, 0xa5, 0x97, 0x12, 0x39, 0x12, 0xf7, 0xe3,
0xaa, 0xd9, 0x0e, 0x38, 0x0f, 0xd7, 0x99, 0xd9, 0xfc, 0xa1, 0x75, 0x2b, 0x0f, 0xed, 0x00, 0x56,
0xcd, 0x26, 0xf8, 0x3f, 0x9d, 0xfe, 0x5a, 0x83, 0x55, 0xa1, 0xb2, 0xf0, 0x95, 0x0a, 0xe2, 0x4c,
0xa7, 0xd3, 0xa1, 0x0e, 0x93, 0x18, 0xed, 0xbf, 0x4c, 0x0e, 0x6c, 0xaa, 0x5d, 0x61, 0x88, 0x37,
0xe9, 0x24, 0xf6, 0x08, 0x5a, 0xb3, 0xdd, 0x3f, 0xaf, 0x5a, 0x55, 0x61, 0x8f, 0x60, 0x69, 0x90,
0x4c, 0xd3, 0x61, 0x31, 0xdb, 0x77, 0x4a, 0x6d, 0x83, 0xcc, 0x88, 0x45, 0xae, 0x56, 0xe9, 0xa3,
0xfa, 0xeb, 0xfb, 0x88, 0x3d, 0x9d, 0xe9, 0x23, 0xbf, 0x41, 0x06, 0xef, 0x94, 0x06, 0x17, 0xc4,
0xe2, 0xa2, 0x36, 0xff, 0xc9, 0x81, 0x1b, 0x55, 0x08, 0x6f, 0x34, 0x18, 0x45, 0x45, 0x6a, 0x0b,
0x2b, 0xe2, 0x2e, 0xaa, 0x88, 0x57, 0x56, 0xa4, 0x7c, 0xbb, 0xeb, 0x95, 0xb7, 0x9b, 0x1f, 0xc1,
0xdd, 0xb9, 0x32, 0x6d, 0x25, 0xe3, 0x09, 0xf6, 0xc3, 0x7f, 0x28, 0x17, 0xae, 0x8c, 0x34, 0xb5,
0x85, 0x6a, 0x0a, 0x43, 0xf0, 0x4f, 0xe1, 0xed, 0x81, 0xd2, 0x95, 0x22, 0xe5, 0xdd, 0xd6, 0x05,
0x77, 0x57, 0x9d, 0x5c, 0x12, 0x3e, 0x8a, 0xf8, 0xe7, 0xe0, 0xef, 0x4f, 0x46, 0x52, 0xab, 0x6b,
0x59, 0x6f, 0xc2, 0xf2, 0x5e, 0x32, 0x49, 0xa2, 0xe4, 0xf0, 0xec, 0x8a, 0x91, 0xf7, 0x61, 0xc9,
0xec, 0x47, 0xf3, 0x19, 0xd9, 0x14, 0x39, 0xc9, 0x6f, 0x61, 0x43, 0x0f, 0x65, 0x34, 0x9c, 0x46,
0x08, 0x03, 0xbf, 0x27, 0xb3, 0xcd, 0xce, 0xef, 0xe7, 0x6b, 0xce, 0x1f, 0xe7, 0x6b, 0xce, 0x9f,
0xe7, 0x6b, 0xce, 0x2f, 0x7f, 0xad, 0xbd, 0x75, 0xd0, 0xa0, 0xbf, 0x86, 0xc7, 0xff, 0x04, 0x00,
0x00, 0xff, 0xff, 0x60, 0xc2, 0x0f, 0x6a, 0x46, 0x0c, 0x00, 0x00,
// 1035 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x56, 0xcb, 0x6f, 0x1b, 0x45,
0x18, 0x67, 0xbd, 0x6b, 0x27, 0xfe, 0x8c, 0x53, 0x67, 0x5a, 0xc2, 0x16, 0xa1, 0x60, 0x46, 0x45,
0x0d, 0x1c, 0xa2, 0x92, 0x5e, 0x78, 0x55, 0x8a, 0x12, 0xa7, 0x62, 0x11, 0x89, 0x60, 0x36, 0xe9,
0x01, 0x89, 0xc3, 0xd4, 0x1e, 0xa5, 0xab, 0xac, 0x77, 0xcc, 0xee, 0x6c, 0x1e, 0x3d, 0x70, 0x85,
0x0b, 0x17, 0x4e, 0x88, 0xbf, 0x88, 0x23, 0x7f, 0x02, 0x0a, 0xff, 0x08, 0x9a, 0x6f, 0x66, 0x1f,
0xf1, 0xa3, 0xa9, 0x4c, 0x6f, 0xfb, 0xbd, 0x5f, 0xbf, 0xef, 0x9b, 0x85, 0xee, 0x24, 0x8d, 0xce,
0xb9, 0x12, 0xdb, 0x93, 0x54, 0x2a, 0x49, 0x56, 0xa3, 0x44, 0x89, 0x34, 0xe1, 0x31, 0xed, 0x40,
0x3b, 0x48, 0x46, 0xe2, 0xf2, 0x50, 0x28, 0x4e, 0x7f, 0x77, 0xa0, 0xfd, 0x34, 0xe5, 0x63, 0xa1,
0x29, 0xf2, 0x3e, 0xb4, 0xf7, 0xf9, 0xf0, 0x85, 0x38, 0xbe, 0x9a, 0x08, 0xdf, 0xed, 0x3b, 0x5b,
0x6d, 0x56, 0x31, 0x4a, 0x69, 0x18, 0xbd, 0x14, 0xbe, 0xd7, 0x77, 0xb6, 0xba, 0xac, 0x62, 0x90,
0x3e, 0x74, 0x8e, 0xa3, 0xb1, 0xf8, 0x3e, 0xe7, 0x89, 0xca, 0xc7, 0x7e, 0x13, 0xad, 0xeb, 0x2c,
0xf2, 0x10, 0x5a, 0x4f, 0x23, 0x11, 0x8f, 0x32, 0x7f, 0xa5, 0xef, 0x6e, 0x75, 0x76, 0xee, 0x6c,
0x17, 0x39, 0x6d, 0x23, 0x9f, 0x59, 0x31, 0xa5, 0xb0, 0x16, 0x8c, 0x27, 0x32, 0x55, 0x4c, 0x64,
0x13, 0x99, 0x64, 0x82, 0xf4, 0xc0, 0x3d, 0x48, 0x53, 0xdf, 0x41, 0xa7, 0xfa, 0x93, 0xfe, 0x0c,
0xbd, 0xbd, 0x58, 0x0e, 0xcf, 0x06, 0x5c, 0x71, 0x26, 0x7e, 0xca, 0x45, 0xa6, 0xc8, 0x3d, 0x68,
0x62, 0x65, 0x56, 0xcf, 0x10, 0x9a, 0x8b, 0x15, 0xfa, 0x0d, 0xc3, 0x45, 0x42, 0x73, 0xd1, 0x1e,
0xcb, 0xf4, 0x98, 0x21, 0x34, 0x37, 0x8c, 0xa3, 0xa1, 0x29, 0xcf, 0x63, 0x86, 0x20, 0x04, 0xbc,
0x67, 0x91, 0xb8, 0xb0, 0x35, 0xe1, 0x37, 0x0d, 0x60, 0xbd, 0x16, 0xdf, 0xa6, 0xb9, 0x01, 0x2d,
0x26, 0x2f, 0x82, 0x41, 0xe6, 0x3b, 0x7d, 0x77, 0xcb, 0x63, 0x96, 0xc2, 0xce, 0xc9, 0x38, 0x1f,
0x27, 0x5a, 0xd4, 0x40, 0x51, 0xc5, 0xa0, 0xf7, 0xa1, 0x89, 0x6d, 0xd4, 0x55, 0x56, 0xb6, 0xfa,
0x93, 0xfe, 0xe2, 0x40, 0xfb, 0x90, 0x5f, 0x62, 0x1a, 0x19, 0x79, 0x02, 0xab, 0xa1, 0xe2, 0xc9,
0x88, 0xa7, 0x23, 0x54, 0xea, 0xec, 0x7c, 0x58, 0xb5, 0xb0, 0x54, 0xdb, 0x2e, 0x74, 0x0e, 0x12,
0x95, 0x5e, 0xb1, 0xd2, 0xe4, 0xbd, 0x2f, 0xa1, 0x7b, 0x43, 0xa4, 0xe3, 0x9d, 0x89, 0xab, 0xa2,
0xab, 0x67, 0xe2, 0x4a, 0xd7, 0x7f, 0xce, 0xe3, 0xdc, 0xf4, 0xca, 0x63, 0x86, 0xf8, 0xa2, 0xf1,
0x99, 0x43, 0x77, 0x81, 0xec, 0xa7, 0x82, 0x2b, 0x81, 0x41, 0x0e, 0x45, 0x96, 0xf1, 0x53, 0xb1,
0xb8, 0xe3, 0xa6, 0x8b, 0x8d, 0x5a, 0x17, 0xe9, 0x27, 0x40, 0x06, 0x22, 0x16, 0x4a, 0x58, 0xf4,
0xbd, 0xc2, 0x03, 0x0d, 0x8b, 0x68, 0xb7, 0xeb, 0x92, 0x87, 0xe0, 0x69, 0xf0, 0x62, 0xb0, 0xce,
0xce, 0xdd, 0xaa, 0x23, 0x25, 0xca, 0x19, 0x2a, 0xd0, 0xa8, 0x70, 0x6a, 0x01, 0x7f, 0x4b, 0x09,
0x73, 0x40, 0x53, 0x84, 0x72, 0xa7, 0x43, 0x95, 0x2b, 0x64, 0x43, 0xed, 0x16, 0xb5, 0x2e, 0x1b,
0x8a, 0x9e, 0x96, 0xc9, 0xea, 0x9d, 0x58, 0x26, 0xd9, 0x8f, 0xa0, 0x89, 0xb6, 0x36, 0xdb, 0x99,
0x6d, 0x33, 0x52, 0xfa, 0xac, 0x4c, 0x75, 0xd9, 0x40, 0xf7, 0xea, 0x81, 0xda, 0x85, 0xdf, 0x1f,
0xac, 0xae, 0xde, 0x9e, 0x23, 0x6d, 0x63, 0x3c, 0xe1, 0xf7, 0xe2, 0x99, 0x4d, 0x35, 0x52, 0xfb,
0xd6, 0xeb, 0x96, 0xf9, 0x6e, 0xdf, 0xd5, 0xbe, 0x91, 0xa0, 0x8f, 0xa1, 0x15, 0x0e, 0x5f, 0x88,
0x31, 0x27, 0x1f, 0xc3, 0x0a, 0xa6, 0x26, 0x32, 0xbb, 0x11, 0x77, 0xa6, 0xe6, 0xcf, 0x0a, 0x39,
0x1d, 0xd8, 0x92, 0x16, 0x24, 0xd4, 0xc2, 0xd0, 0x99, 0xef, 0xcd, 0xdc, 0x26, 0xcd, 0x67, 0x56,
0x4c, 0x0f, 0xc0, 0x3d, 0x61, 0x81, 0xde, 0x74, 0xcc, 0xa0, 0xf0, 0x62, 0x29, 0xed, 0xfb, 0x6b,
0x99, 0x29, 0xdb, 0x20, 0xfc, 0xd6, 0xbc, 0xef, 0x64, 0xaa, 0xb0, 0x3d, 0x5d, 0x86, 0xdf, 0xf4,
0x47, 0xf0, 0x8e, 0xe4, 0x48, 0x90, 0x35, 0x68, 0x04, 0x03, 0xeb, 0xa3, 0x11, 0x0c, 0xc8, 0x07,
0xe8, 0xde, 0xf6, 0xa5, 0x5b, 0x25, 0x71, 0xc2, 0x02, 0x86, 0x81, 0x1f, 0x40, 0x37, 0xc8, 0xf6,
0xa5, 0x4c, 0x47, 0x51, 0xc2, 0x95, 0x4c, 0xd1, 0xeb, 0x2a, 0xbb, 0xc9, 0xa4, 0xbb, 0xd0, 0xd3,
0xee, 0x43, 0xc5, 0x55, 0x89, 0xbe, 0x0d, 0x68, 0x69, 0x5e, 0x19, 0xce, 0x52, 0xb8, 0xad, 0x5a,
0xaf, 0x18, 0x2a, 0x12, 0xf4, 0x5b, 0xe3, 0xe1, 0xe0, 0x5c, 0x24, 0xaa, 0x06, 0x0a, 0xa4, 0xd1,
0x41, 0x97, 0x19, 0x82, 0x50, 0x53, 0x8a, 0xcd, 0x79, 0xad, 0xca, 0x59, 0x73, 0x19, 0xca, 0xe8,
0x6f, 0x0e, 0x40, 0x91, 0x50, 0x9e, 0x95, 0x26, 0xce, 0x62, 0x13, 0xf2, 0x69, 0xed, 0xf2, 0xcd,
0xe2, 0xa4, 0x14, 0xb1, 0xda, 0x7d, 0xdc, 0x2a, 0x60, 0x61, 0x21, 0xdf, 0xab, 0xf4, 0x0d, 0xdf,
0x8e, 0x49, 0x9f, 0x82, 0xee, 0x7e, 0x9c, 0x67, 0x4a, 0xa4, 0x36, 0x23, 0x7d, 0xa1, 0x0d, 0xa3,
0xec, 0x4f, 0xc5, 0x98, 0xdf, 0x22, 0xf2, 0x00, 0x9a, 0x3a, 0x53, 0x83, 0xcd, 0xd9, 0x32, 0x8c,
0x90, 0x86, 0x76, 0x3b, 0xe6, 0xc2, 0x8e, 0x80, 0x87, 0x6f, 0xad, 0x85, 0x0b, 0x3e, 0xb3, 0x3d,
0x70, 0x0f, 0xa3, 0x04, 0x4b, 0x70, 0x99, 0xfe, 0x44, 0x0e, 0xbf, 0xc4, 0x37, 0x49, 0x73, 0xb8,
0xbe, 0x8f, 0xeb, 0xe6, 0x3a, 0xe8, 0x7d, 0x58, 0x66, 0x67, 0x8b, 0x27, 0xcd, 0xad, 0x3d, 0x69,
0x21, 0xac, 0x9b, 0x4b, 0xf0, 0x26, 0x9d, 0xfe, 0xd9, 0x80, 0x75, 0x26, 0xb2, 0xe8, 0xa5, 0x08,
0x92, 0x4c, 0xa5, 0xf9, 0x50, 0x45, 0x32, 0xd1, 0xf6, 0xdf, 0xc8, 0xe7, 0xb6, 0xd5, 0x2e, 0x33,
0xc4, 0xeb, 0x20, 0x89, 0x3c, 0x82, 0xce, 0x34, 0xfa, 0x67, 0x55, 0xeb, 0x2a, 0xe4, 0x11, 0xac,
0x84, 0x32, 0x4f, 0x87, 0xe5, 0x6e, 0x6f, 0x54, 0xda, 0x26, 0x33, 0x23, 0x66, 0x85, 0x5a, 0x0d,
0x47, 0xcd, 0x57, 0xe3, 0x88, 0x3c, 0x99, 0xc2, 0x91, 0xdf, 0x42, 0x83, 0x77, 0x2b, 0x83, 0x1b,
0x62, 0x76, 0x53, 0x9b, 0xfe, 0xea, 0xc0, 0xdb, 0xf5, 0x14, 0x5e, 0x6b, 0x31, 0xca, 0x89, 0x34,
0xe6, 0x4e, 0xc4, 0x9d, 0x37, 0x11, 0xaf, 0x9a, 0x48, 0xf5, 0x3a, 0x37, 0xeb, 0xaf, 0xf3, 0x19,
0xdc, 0x9f, 0x19, 0xd3, 0xbe, 0x1c, 0x4f, 0x34, 0x1e, 0xfe, 0xc7, 0xb8, 0xf4, 0xc9, 0x48, 0x53,
0x3b, 0xa8, 0x36, 0x33, 0x04, 0xfd, 0x1c, 0xde, 0x09, 0x85, 0xaa, 0x0d, 0xa9, 0x40, 0x5b, 0x1f,
0xdc, 0x23, 0x71, 0xb1, 0xa0, 0x7c, 0x2d, 0xa2, 0x5f, 0x81, 0x7f, 0x32, 0x19, 0x71, 0x25, 0x96,
0xb2, 0xde, 0x83, 0xd5, 0x63, 0x39, 0x91, 0xb1, 0x3c, 0xbd, 0xba, 0x65, 0xe5, 0x7d, 0x58, 0x31,
0xf7, 0xd1, 0xfc, 0xb0, 0xb5, 0x59, 0x41, 0xd2, 0xbb, 0x1a, 0xd0, 0x43, 0x1e, 0x0f, 0xf3, 0x58,
0xa7, 0xa1, 0xff, 0xdc, 0xb2, 0xbd, 0xde, 0x5f, 0xd7, 0x9b, 0xce, 0xdf, 0xd7, 0x9b, 0xce, 0x3f,
0xd7, 0x9b, 0xce, 0x1f, 0xff, 0x6e, 0xbe, 0xf5, 0xbc, 0x85, 0xff, 0xdd, 0x8f, 0xff, 0x0b, 0x00,
0x00, 0xff, 0xff, 0xd3, 0x15, 0x68, 0xea, 0x88, 0x0b, 0x00, 0x00,
}

3
internal/private.proto
View file

@ -6,7 +6,6 @@ message IndexMeta {
}
message FrameMeta {
bool InverseEnabled = 2;
string CacheType = 3;
uint32 CacheSize = 4;
string TimeQuantum = 5;
@ -36,13 +35,11 @@ message Cache {
message MaxSlices {
map<string, uint64> Standard = 1;
map<string, uint64> Inverse = 2;
}
message CreateSliceMessage {
string Index = 1;
uint64 Slice = 2;
bool IsInverse = 3;
}
message DeleteIndexMessage {

7
pilosa.go
View file

@ -32,10 +32,9 @@ var (
ErrIndexNotFound = errors.New("index not found")
// ErrFrameRequired is returned when no frame is specified.
ErrFrameRequired = errors.New("frame required")
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
ErrFrameInverseDisabled = errors.New("frame inverse disabled")
ErrFrameRequired = errors.New("frame required")
ErrFrameExists = errors.New("frame already exists")
ErrFrameNotFound = errors.New("frame not found")
ErrFieldNotFound = errors.New("field not found")
ErrFieldExists = errors.New("field already exists")

28
pql/ast.go
View file

@ -177,34 +177,6 @@ func (c *Call) String() string {
return buf.String()
}
// SupportsInverse indicates that the call may be on an inverse frame.
func (c *Call) SupportsInverse() bool {
return c.Name == "Bitmap" || c.Name == "TopN"
}
// IsInverse specifies if the call is for an inverse view.
// Return defaults to false unless absolutely sure of inversion.
func (c *Call) IsInverse(rowLabel, columnLabel string) bool {
if c.SupportsInverse() {
// Top-n has an explicit inverse flag.
if c.Name == "TopN" {
inverse, _ := c.Args["inverse"].(bool)
return inverse
}
// Bitmap calls use the row/column labels to determine whether inverse.
_, rowOK, rowErr := c.UintArg(rowLabel)
_, columnOK, columnErr := c.UintArg(columnLabel)
if rowErr != nil || columnErr != nil {
return false
}
if !rowOK && columnOK {
return true
}
}
return false
}
// HasConditionArg returns true if any arg is a conditional.
func (c *Call) HasConditionArg() bool {
for _, v := range c.Args {

66
pql/ast_test.go
View file

@ -67,69 +67,3 @@ func TestCondition_Value(t *testing.T) {
}
})
}
// Ensure call can be converted into a string.
func TestCall_SupportsInverse(t *testing.T) {
t.Run("Bitmap", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap()`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].SupportsInverse() != true {
t.Fatalf("call should support inverse: %s", q.Calls[0])
}
})
t.Run("Count Bitmap", func(t *testing.T) {
q, err := pql.ParseString(`Count(Bitmap())`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].SupportsInverse() == true {
t.Fatalf("call should not support inverse: %s", q.Calls[0])
}
})
t.Run("Union Bitmaps", func(t *testing.T) {
q, err := pql.ParseString(`Union(Bitmap(), Bitmap())`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].SupportsInverse() == true {
t.Fatalf("call should not support inverse: %s", q.Calls[0])
}
})
}
// Ensure call is correctly determined to be against an inverse view.
func TestCall_IsInverse(t *testing.T) {
t.Run("Bitmap Row", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap(frame="f", row=1)`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("row", "col") != false {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
t.Run("Bitmap Column", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap(frame="f", col=1)`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("row", "col") != true {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
t.Run("Bitmap Column No Label", func(t *testing.T) {
q, err := pql.ParseString(`Bitmap(frame="f", col=1)`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("rowX", "colX") != false {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
t.Run("Count", func(t *testing.T) {
q, err := pql.ParseString(`Count(Bitmap(frame="f", col=1))`)
if err != nil {
t.Fatal(err)
} else if q.Calls[0].IsInverse("row", "col") != false {
t.Fatalf("incorrect call inverse: %s", q.Calls[0])
}
})
}

26
server.go
View file

@ -440,11 +440,7 @@ func (s *Server) ReceiveMessage(pb proto.Message) error {
if idx == nil {
return fmt.Errorf("Local Index not found: %s", obj.Index)
}
if obj.IsInverse {
idx.SetRemoteMaxInverseSlice(obj.Slice)
} else {
idx.SetRemoteMaxSlice(obj.Slice)
}
idx.SetRemoteMaxSlice(obj.Slice)
case *internal.CreateIndexMessage:
opt := IndexOptions{}
_, err := s.Holder.CreateIndex(obj.Index, opt)
@ -569,7 +565,7 @@ func (s *Server) SendTo(to *Node, pb proto.Message) error {
// where a node fails to receive a Broadcast message, or
// when a new (empty) node needs to get in sync with the
// rest of the cluster, two things are shared via gossip:
// - MaxSlice/MaxInverseSlice by Index
// - MaxSlice by Index
// - Schema
// In a gossip implementation, memberlist.Delegate.LocalState() uses this.
func (s *Server) LocalStatus() (proto.Message, error) {
@ -625,7 +621,7 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
return errors.Wrap(err, "applying schema")
}
// Sync maxSlices (standard).
// Sync maxSlices.
oldmaxslices := s.Holder.MaxSlices()
for index, newMax := range ns.MaxSlices.Standard {
localIndex := s.Holder.Index(index)
@ -641,22 +637,6 @@ func (s *Server) mergeRemoteStatus(ns *internal.NodeStatus) error {
}
}
// Sync maxSlices (inverse).
oldMaxInverseSlices := s.Holder.MaxInverseSlices()
for index, newMaxInverse := range ns.MaxSlices.Inverse {
localIndex := s.Holder.Index(index)
// if we don't know about an index locally, log an error because
// indexes should be created and synced prior to slice creation
if localIndex == nil {
s.logger.Printf("Local Index not found: %s", index)
continue
}
if newMaxInverse > oldMaxInverseSlices[index] {
oldMaxInverseSlices[index] = newMaxInverse
localIndex.SetRemoteMaxInverseSlice(newMaxInverse)
}
}
return nil
}

126
server/server_test.go
View file

@ -26,7 +26,6 @@ import (
"strings"
"testing"
"testing/quick"
"time"
"github.com/BurntSushi/toml"
"github.com/pilosa/pilosa"
@ -69,8 +68,8 @@ func TestMain_Set_Quick(t *testing.T) {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"columns": columnIDs,
"attrs": map[string]interface{}{},
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
@ -92,8 +91,8 @@ func TestMain_Set_Quick(t *testing.T) {
exp := MustMarshalJSON(map[string]interface{}{
"results": []interface{}{
map[string]interface{}{
"columns": columnIDs,
"attrs": map[string]interface{}{},
"columns": columnIDs,
"attrs": map[string]interface{}{},
},
},
}) + "\n"
@ -236,123 +235,6 @@ func TestMain_SetColumnAttrs(t *testing.T) {
}
}
// Ensure inverse slices get handled correctly in a multi-node query.
func TestMain_InverseSlices(t *testing.T) {
mains := test.MustRunMainWithCluster(t, 2)
m0 := mains[0]
m1 := mains[1]
// Make sure to use node0 in the cluster.
var m *test.Main
if m0.Server.NodeID < m1.Server.NodeID {
m = m0
} else {
m = m1
}
// Create frames.
client := m.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal("create frame:", err)
}
// Write data on cluster.
if _, err := m.Query("i", "", fmt.Sprintf(`
SetBit(col=1, frame="f", row=1000)
SetBit(col=1, frame="f", row=2000)
SetBit(col=1, frame="f", row=%d)
`, 1*pilosa.SliceWidth)); err != nil {
t.Fatal("setting columns:", err)
}
time.Sleep(1 * time.Second)
// Query the cluster.
if res, err := m.Query("i", "", `Bitmap(col=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != fmt.Sprintf(`{"results":[{"attrs":{},"columns":[1000,2000,%d]}]}`, 1*pilosa.SliceWidth)+"\n" {
t.Fatalf("unexpected result: %s", res)
}
}
// Ensure program can set columns on one cluster and then restore to a second cluster.
func TestMain_FrameRestore(t *testing.T) {
mains1 := test.MustRunMainWithCluster(t, 2)
m10 := mains1[0]
m11 := mains1[1]
// Create frames.
client := m10.Client()
if err := client.CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create index:", err)
}
if err := client.CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal("create frame:", err)
}
// Write data on first cluster.
if _, err := m10.Query("i", "", `
SetBit(row=1, frame="f", col=100)
SetBit(row=1, frame="f", col=1000)
SetBit(row=1, frame="f", col=100000)
SetBit(row=1, frame="f", col=200000)
SetBit(row=1, frame="f", col=400000)
SetBit(row=1, frame="f", col=600000)
SetBit(row=1, frame="f", col=800000)
`); err != nil {
t.Fatal("setting columns:", err)
}
// Query row on first cluster.
if res, err := m10.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("unexpected result: %s", res)
}
// Start second cluster.
mains2 := test.MustRunMainWithCluster(t, 2)
m20 := mains2[0]
defer m20.Close()
m21 := mains2[1]
defer m21.Close()
// Import from first cluster.
client20, err := pilosa.NewInternalHTTPClient(m20.Server.URI.HostPort(), server.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
client21, err := pilosa.NewInternalHTTPClient(m21.Server.URI.HostPort(), server.GetHTTPClient(nil))
if err != nil {
t.Fatal("new client:", err)
}
if err := m20.Client().CreateIndex(context.Background(), "i", pilosa.IndexOptions{}); err != nil && err != pilosa.ErrIndexExists {
t.Fatal("create new index:", err)
}
if err := m20.Client().CreateFrame(context.Background(), "i", "f", pilosa.FrameOptions{}); err != nil {
t.Fatal("create new frame:", err)
}
if err := client20.RestoreFrame(context.Background(), m10.Server.URI.HostPort(), "i", "f"); err != nil {
t.Fatal("restore frame:", err)
}
if err := client21.RestoreFrame(context.Background(), m11.Server.URI.HostPort(), "i", "f"); err != nil {
t.Fatal("restore frame:", err)
}
// Query row on second cluster.
if res, err := m20.Query("i", "", `Bitmap(row=1, frame="f")`); err != nil {
t.Fatal("another bitmap query:", err)
} else if res != `{"results":[{"attrs":{},"columns":[100,1000,100000,200000,400000,600000,800000]}]}`+"\n" {
t.Fatalf("2unexpected result: %s", res)
}
}
// Ensure the host can be parsed.
func TestConfig_Parse_Host(t *testing.T) {
if c, err := ParseConfig(`bind = "local"`); err != nil {

13
view.go
View file

@ -30,14 +30,13 @@ import (
// View layout modes.
const (
ViewStandard = "standard"
ViewInverse = "inverse"
ViewFieldPrefix = "field_"
)
// IsValidView returns true if name is valid.
func IsValidView(name string) bool {
return name == ViewStandard || name == ViewInverse
return name == ViewStandard
}
// View represents a container for frame data.
@ -252,9 +251,8 @@ func (v *View) createFragmentIfNotExists(slice uint64) (*Fragment, error) {
// Send the create slice message to all nodes.
err := v.broadcaster.SendAsync(
&internal.CreateSliceMessage{
Index: v.index,
Slice: slice,
IsInverse: IsInverseView(v.name),
Index: v.index,
Slice: slice,
})
if err != nil {
return nil, errors.Wrap(err, "sending message")
@ -428,11 +426,6 @@ func (v *View) FieldRangeBetween(bitDepth uint, predicateMin, predicateMax uint6
return r, nil
}
// IsInverseView returns true if the view is used for storing an inverted representation.
func IsInverseView(name string) bool {
return strings.HasPrefix(name, ViewInverse)
}
// ViewInfo represents schema information for a view.
type ViewInfo struct {
Name string `json:"name"`

3
webui/assets/main.js
View file

@ -581,14 +581,11 @@ function parse_query(query, indexname) {
function parse_options(option_str) {
var int_keys = ["cacheSize"];
var bool_keys = ["inverseEnabled"];
var options = {};
for (var i = 0; i < option_str.length; i++) {
var parts = option_str[i].split('=');
if (int_keys.indexOf(parts[0]) !== -1 ){
options[parts[0]] = Number(parts[1])
} else if (bool_keys.indexOf(parts[0]) !== -1){
options[parts[0]] = (parts[1] == "true")
} else {
options[parts[0]] = parts[1]
}