scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-08-28 10:54:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge remote-tracking branch 'origin/master' into input-definition

Browse source 
Conflicts:
	handler_test.go
	index_test.go
This commit is contained in:
Michael Baird 2017-06-29 14:32:50 -05:00
commit 504d25b36f
33 changed files with 1187 additions and 1198 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
Makefile
View file

@ -48,7 +48,7 @@ cover-pkg:
mkdir -p build/coverage
touch build/coverage/$(subst /,-,$(PKG)).out
go test -coverprofile=build/coverage/$(subst /,-,$(PKG)).out $(PKG)
tail +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out
tail -n +2 build/coverage/$(subst /,-,$(PKG)).out >> build/coverage/all.out
cover-viz: cover
go tool cover -html=build/coverage/all.out

78
attr_test.go
View file

@ -15,19 +15,15 @@
package pilosa_test
import (
"io/ioutil"
"os"
"reflect"
"runtime"
"sync"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// Ensure database can set and retrieve column attributes.
func TestAttrStore_Attrs(t *testing.T) {
s := MustOpenAttrStore()
s := test.MustOpenAttrStore()
defer s.Close()
// Set attributes.
@ -56,7 +52,7 @@ func TestAttrStore_Attrs(t *testing.T) {
// Ensure database returns a non-nil empty map if unset.
func TestAttrStore_Attrs_Empty(t *testing.T) {
s := MustOpenAttrStore()
s := test.MustOpenAttrStore()
defer s.Close()
if m, err := s.Attrs(100); err != nil {
@ -68,7 +64,7 @@ func TestAttrStore_Attrs_Empty(t *testing.T) {
// Ensure database can unset attributes if explicitly set to nil.
func TestAttrStore_Attrs_Unset(t *testing.T) {
s := MustOpenAttrStore()
s := test.MustOpenAttrStore()
defer s.Close()
// Set attributes.
@ -88,7 +84,7 @@ func TestAttrStore_Attrs_Unset(t *testing.T) {
// Ensure attribute block checksums can be returned.
func TestAttrStore_Blocks(t *testing.T) {
s := MustOpenAttrStore()
s := test.MustOpenAttrStore()
defer s.Close()
// Set attributes.
@ -127,67 +123,3 @@ func TestAttrStore_Blocks(t *testing.T) {
t.Fatalf("block 2 mismatch: %#v != %#v", blks0[2], blks1[2])
}
}
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
*pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore() *AttrStore {
f, err := ioutil.TempFile("", "pilosa-attr-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
const n = 5
for i := 0; i < n; i++ {
if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
b.Fatal(err)
}
}
b.ReportAllocs()
b.ResetTimer()
// Update attributes with an existing subset.
cpuN := runtime.GOMAXPROCS(0)
var wg sync.WaitGroup
for i := 0; i < cpuN; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < b.N/cpuN; j++ {
if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
b.Fatal(err)
}
}
}()
}
wg.Wait()
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore() *AttrStore {
s := NewAttrStore()
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the database and removes the underlying data.
func (s *AttrStore) Close() error {
defer os.RemoveAll(s.Path())
return s.AttrStore.Close()
}

83
client_test.go
View file

@ -25,15 +25,16 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
func createCluster(c *pilosa.Cluster) ([]*Server, []*Holder) {
func createCluster(c *pilosa.Cluster) ([]*test.Server, []*test.Holder) {
numNodes := len(c.Nodes)
hldr := make([]*Holder, numNodes)
server := make([]*Server, numNodes)
hldr := make([]*test.Holder, numNodes)
server := make([]*test.Server, numNodes)
for i := 0; i < numNodes; i++ {
hldr[i] = MustOpenHolder()
server[i] = NewServer()
hldr[i] = test.MustOpenHolder()
server[i] = test.NewServer()
server[i].Handler.Host = server[i].Host()
server[i].Handler.Cluster = c
server[i].Handler.Cluster.Nodes[i].Host = server[i].Host()
@ -44,7 +45,7 @@ func createCluster(c *pilosa.Cluster) ([]*Server, []*Holder) {
// Test distributed TopN Row count across 3 nodes.
func TestClient_MultiNode(t *testing.T) {
cluster := NewCluster(3)
cluster := test.NewCluster(3)
s, hldr := createCluster(cluster)
for i := 0; i < len(cluster.Nodes); i++ {
@ -130,10 +131,10 @@ func TestClient_MultiNode(t *testing.T) {
hldr[2].MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, sliceNums[2]).RecalculateCache()
// Connect to each node to compare results.
client := make([]*Client, 3)
client[0] = MustNewClient(s[0].Host())
client[1] = MustNewClient(s[1].Host())
client[2] = MustNewClient(s[2].Host())
client := make([]*test.Client, 3)
client[0] = test.MustNewClient(s[0].Host())
client[1] = test.MustNewClient(s[1].Host())
client[2] = test.MustNewClient(s[2].Host())
topN := 4
q := fmt.Sprintf(`TopN(frame="%s", n=%d)`, "f", topN)
@ -197,22 +198,22 @@ func TestClient_MultiNode(t *testing.T) {
// Ensure client can bulk import data.
func TestClient_Import(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
// Load bitmap into cache to ensure cache gets updated.
f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
f.Row(0)
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
// Send import request.
c := MustNewClient(s.Host())
c := test.MustNewClient(s.Host())
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
@ -232,7 +233,7 @@ func TestClient_Import(t *testing.T) {
// Ensure client can bulk import data to an inverse frame.
func TestClient_ImportInverseEnabled(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
@ -255,15 +256,15 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
// Load bitmap into cache to ensure cache gets updated.
f.Row(0)
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
// Send import request.
c := MustNewClient(s.Host())
c := test.MustNewClient(s.Host())
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
@ -287,7 +288,7 @@ func TestClient_ImportInverseEnabled(t *testing.T) {
// Ensure client backup and restore a frame.
func TestClient_BackupRestore(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
@ -295,14 +296,14 @@ func TestClient_BackupRestore(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).MustSetBits(100, (5*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(200, 20000)
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
c := MustNewClient(s.Host())
c := test.MustNewClient(s.Host())
// Backup from frame.
var buf bytes.Buffer
@ -335,7 +336,7 @@ func TestClient_BackupRestore(t *testing.T) {
// Ensure client backup and restore a frame with inverse view.
func TestClient_BackupInverseView(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
@ -360,14 +361,14 @@ func TestClient_BackupInverseView(t *testing.T) {
f.SetBit(100, 3)
f.SetBit(100, SliceWidth-1)
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
c := MustNewClient(s.Host())
c := test.MustNewClient(s.Host())
// Backup from frame.
var buf bytes.Buffer
@ -392,19 +393,19 @@ func TestClient_BackupInverseView(t *testing.T) {
// backup returns error with invalid view
func TestClient_BackupInvalidView(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(100, 1, 2, 3, SliceWidth-1)
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
c := MustNewClient(s.Host())
c := test.MustNewClient(s.Host())
// Backup from frame.
var buf bytes.Buffer
@ -416,7 +417,7 @@ func TestClient_BackupInvalidView(t *testing.T) {
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
func TestClient_FragmentBlocks(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set two bits on blocks 0 & 3.
@ -426,15 +427,15 @@ func TestClient_FragmentBlocks(t *testing.T) {
// Set a bit on a different slice.
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(0, 1)
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
// Retrieve blocks.
c := MustNewClient(s.Host())
c := test.MustNewClient(s.Host())
blocks, err := c.FragmentBlocks(context.Background(), "i", "f", pilosa.ViewStandard, 0)
if err != nil {
t.Fatal(err)
@ -451,17 +452,3 @@ func TestClient_FragmentBlocks(t *testing.T) {
t.Fatalf("blocks mismatch:\n\nexp=%s\n\ngot=%s\n\n", spew.Sdump(a), spew.Sdump(blocks))
}
}
// Client represents a test wrapper for pilosa.Client.
type Client struct {
*pilosa.Client
}
// MustNewClient returns a new instance of Client. Panic on error.
func MustNewClient(host string) *Client {
c, err := pilosa.NewClient(host)
if err != nil {
panic(err)
}
return &Client{Client: c}
}

39
cluster_test.go
View file

@ -15,7 +15,6 @@
package pilosa_test
import (
"fmt"
"math/rand"
"reflect"
"testing"
@ -24,6 +23,7 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/httpbroadcast"
"github.com/pilosa/pilosa/test"
)
// Ensure the cluster can fairly distribute partitions across the nodes.
@ -34,7 +34,7 @@ func TestCluster_Owners(t *testing.T) {
{Host: "serverB:1000"},
{Host: "serverC:1000"},
},
Hasher: NewModHasher(),
Hasher: test.NewModHasher(),
ReplicaN: 2,
}
@ -134,43 +134,10 @@ func TestCluster_NodeStates(t *testing.T) {
// Ensure OwnsSlices can find the actual slice list for node and index
func TestCluster_OwnsSlices(t *testing.T) {
c := NewCluster(5)
c := test.NewCluster(5)
slices := c.OwnsSlices("test", 10, "host2")
if !reflect.DeepEqual(slices, []uint64{0, 3, 6, 10}) {
t.Fatalf("unexpected slices for node's index: %v", slices)
}
}
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewCluster(n int) *pilosa.Cluster {
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
Host: fmt.Sprintf("host%d", i),
})
}
return c
}
// ModHasher represents a simple, mod-based hashing.
type ModHasher struct{}
// NewModHasher returns a new instance of ModHasher with n buckets.
func NewModHasher() *ModHasher { return &ModHasher{} }
func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
// ConstHasher represents hash that always returns the same index.
type ConstHasher struct {
i int
}
// NewConstHasher returns a new instance of ConstHasher that always returns i.
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }

12
cmd/server.go
View file

@ -17,6 +17,7 @@ package cmd
import (
"fmt"
"io"
"log"
"os"
"os/signal"
"runtime/pprof"
@ -44,7 +45,12 @@ It will load existing data from the configured
directory, and start listening client connections
on the configured port.`,
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Fprintf(Server.Stderr, "Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime)
logOutput, err := server.GetLogWriter(Server.Config.LogPath, stderr)
if err != nil {
return err
}
logger := log.New(logOutput, "", log.LstdFlags)
logger.Printf("Pilosa %s, build time %s\n", pilosa.Version, pilosa.BuildTime)
// Start CPU profiling.
if Server.CPUProfile != "" {
@ -73,7 +79,7 @@ on the configured port.`,
signal.Notify(c, os.Interrupt)
select {
case sig := <-c:
fmt.Fprintf(Server.Stderr, "Received %s; gracefully shutting down...\n", sig.String())
logger.Printf("Received %s; gracefully shutting down...\n", sig.String())
// Second signal causes a hard shutdown.
go func() { <-c; os.Exit(1) }()
@ -82,7 +88,7 @@ on the configured port.`,
return err
}
case <-Server.Done:
fmt.Fprintf(Server.Stderr, "Server closed externally")
logger.Printf("Server closed externally")
}
return nil
},

144
ctl/backup_test.go
View file

@ -17,13 +17,11 @@ package ctl
import (
"bytes"
"context"
"fmt"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"io/ioutil"
"net/http/httptest"
"net/url"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
func TestBackupCommand_FileRequired(t *testing.T) {
@ -43,13 +41,13 @@ func TestBackupCommand_Run(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
cm := NewBackupCommand(stdin, stdout, stderr)
@ -65,133 +63,3 @@ func TestBackupCommand_Run(t *testing.T) {
t.Fatalf("Command not working, error: '%s'", err)
}
}
// Server represents a test wrapper for httptest.Server.
type Server struct {
*httptest.Server
Handler *Handler
}
// NewServer returns a test server running on a random port.
func NewServer() *Server {
s := &Server{
Handler: NewHandler(),
}
s.Server = httptest.NewServer(s.Handler.Handler)
// Update handler to use hostname.
s.Handler.Host = s.Host()
// Handler test messages can no-op.
s.Handler.Broadcaster = pilosa.NopBroadcaster
// Create a default cluster on the handler
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
return s
}
// Handler represents a test wrapper for pilosa.Handler.
type Handler struct {
*pilosa.Handler
Executor HandlerExecutor
}
// NewHandler returns a new instance of Handler.
func NewHandler() *Handler {
h := &Handler{
Handler: pilosa.NewHandler(),
}
h.Handler.Executor = &h.Executor
h.Handler.LogOutput = ioutil.Discard
// Handler test messages can no-op.
h.Broadcaster = pilosa.NopBroadcaster
return h
}
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return c.ExecuteFn(ctx, index, query, slices, opt)
}
// Host returns the hostname of the running server.
func (s *Server) Host() string { return MustParseURLHost(s.URL) }
// MustParseURLHost parses rawurl and returns the hostname. Panic on error.
func MustParseURLHost(rawurl string) string {
u, err := url.Parse(rawurl)
if err != nil {
panic(err)
}
return u.Host
}
func NewCluster(n int) *pilosa.Cluster {
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
Host: fmt.Sprintf("host%d", i),
})
}
return c
}
// ModHasher represents a simple, mod-based hashing.
type ModHasher struct{}
// NewModHasher returns a new instance of ModHasher with n buckets.
func NewModHasher() *ModHasher { return &ModHasher{} }
func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
// ConstHasher represents hash that always returns the same index.
type ConstHasher struct {
i int
}
// NewConstHasher returns a new instance of ConstHasher that always returns i.
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }
// Holder is a test wrapper for pilosa.Holder.
type Holder struct {
*pilosa.Holder
LogOutput bytes.Buffer
}
// NewHolder returns a new instance of Holder with a temporary path.
func NewHolder() *Holder {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
h := &Holder{Holder: pilosa.NewHolder()}
h.Path = path
h.Holder.LogOutput = &h.LogOutput
return h
}
// MustOpenHolder creates and opens a holder at a temporary path. Panic on error.
func MustOpenHolder() *Holder {
h := NewHolder()
if err := h.Open(); err != nil {
panic(err)
}
return h
}

3
ctl/bench_test.go
View file

@ -18,10 +18,11 @@ import (
"bytes"
"context"
"fmt"
"github.com/pilosa/pilosa"
"io"
"os"
"testing"
"github.com/pilosa/pilosa"
)
func TestBenchCommand_InvalidOption(t *testing.T) {

16
ctl/export_test.go
View file

@ -16,11 +16,13 @@ package ctl
import (
"bytes"
"github.com/pilosa/pilosa"
"golang.org/x/net/context"
"net/http"
"strings"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
"golang.org/x/net/context"
)
func TestExportCommand_Validation(t *testing.T) {
@ -53,18 +55,18 @@ func TestExportCommand_Run(t *testing.T) {
stdin, stdout, stderr := GetIO(buf)
cm := NewExportCommand(stdin, stdout, stderr)
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", strings.NewReader("")))
cm.Index = "i"
cm.Frame = "f"

12
ctl/import_test.go
View file

@ -16,13 +16,15 @@ package ctl
import (
"bytes"
"github.com/pilosa/pilosa"
"golang.org/x/net/context"
"io"
"io/ioutil"
"net/http"
"strings"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
"golang.org/x/net/context"
)
func TestImportCommand_Validation(t *testing.T) {
@ -59,12 +61,12 @@ func TestImportCommand_Run(t *testing.T) {
t.Fatal(err)
}
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder
cm.Host = s.Host()

12
ctl/restore_test.go
View file

@ -17,11 +17,13 @@ package ctl
import (
"bufio"
"bytes"
"github.com/pilosa/pilosa"
"golang.org/x/net/context"
"io"
"io/ioutil"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
"golang.org/x/net/context"
)
func TestRestoreCommand_FileRequired(t *testing.T) {
@ -41,13 +43,13 @@ func TestRestoreCommand_Run(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Host = s.Host()
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster = test.NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
s.Handler.Holder = hldr.Holder

50
docs/api-reference.md
View file

@ -5,9 +5,9 @@ title = "API Reference"
## API Reference
### `/index`
### List all index schemas
#### `GET`
`GET /index`
Returns the schema of all indexes in JSON.
@ -21,9 +21,9 @@ Response:
{"indexes":[{"name":"user","frames":[{"name":"collab"}]}]}
```
### `/index/<index-name>`
### List index schema
#### `GET`
`GET /index/<index-name>`
Returns the schema of the specified index in JSON.
@ -37,7 +37,9 @@ Response:
{"index":{"name":"user"}, "frames":[{"name":"collab"}]}]}
```
#### `POST`
### Create index
`POST /index/<index-name>`
Creates an index with the given name.
@ -57,7 +59,9 @@ Response:
{}
```
#### `DELETE`
### Remove index
`DELETE /index/index-name`
Removes the given index.
@ -71,9 +75,9 @@ Response:
{}
```
### `/index/<index-name>/query`
### Query index
#### `POST`
`POST /index/<index-name>/query`
Sends a query to the Pilosa server with the given index. The request body is UTF-8 encoded text and response body is in JSON by default.
@ -91,11 +95,13 @@ Response:
In order to send protobuf binaries in the request and response, set `Content-Type` and `Accept` headers to: `application/x-protobuf`.
The response doesn't include column attributes by default. To return them, set `columnAttrs` query argument to `true`.
The response doesn't include column attributes by default. To return them, set the `columnAttrs` query argument to `true`.
The query is executed for all [slices](../data-model#slice) by default. To use specified slices only, set the `slices` query argument to a comma-separated list of slice indices.
Request:
```
curl localhost:10101/index/user/query?columnAttrs=true \
curl "localhost:10101/index/user/query?columnAttrs=true&slices=0,1" \
-X POST \
-d 'Bitmap(frame="language", id=5)'
```
@ -107,9 +113,9 @@ Response:
}
```
### `/index/<index-name>/time-quantum`
### Change index time quantum
#### `PATCH`
`PATCH /index/<index-name>/time-quantum`
Changes the time quantum for the given index. This endpoint should be called at most once right after creating a database.
@ -139,9 +145,9 @@ Response:
{}
```
### `/index/<index-name>/frame/<frame-name>`
### Create frame
#### `POST`
`POST /index/<index-name>/frame/<frame-name>`
Creates a frame in the given index with the given name.
@ -165,7 +171,9 @@ Response:
{}
```
#### `DELETE`
### Remove frame
`DELETE POST /index/<index-name>/frame/<frame-name>`
Removes the given frame.
@ -179,9 +187,9 @@ Response:
{}
```
### `/index/<index-name>/frame/<frame-name>/time-quantum`
### Change frame time quantum
#### `PATCH`
`PATCH /index/<index-name>/frame/<frame-name>/time-quantum`
Changes the time quantum for the given frame. This endpoint should be called at most once right after creating a frame.
@ -211,9 +219,9 @@ Response:
{}
```
### `/hosts`
### List hosts
#### `GET`
`GET /hosts`
Returns the hosts in the cluster.
@ -227,9 +235,9 @@ Response:
[{"host":":10101","internalHost":""}]
```
### `/version`
### Get version
#### `GET`
`GET /version`
Returns the version of the Pilosa server.

2
docs/data-model.md
View file

@ -54,7 +54,7 @@ Attributes are arbitrary key/value pairs that can be associated to both rows or
### Slice
Indexes are sharded into groups of columns called Slices - each Slice contains a fixed number of columns which is the SliceWidth.
Indexes are sharded into groups of columns called Slices - each Slice contains a fixed number of columns which is the SliceWidth. SliceWidth is a non-configurable constant set to 2<sup>20</sup>.
Columns are sharded on a preset width, and each shard is referred to as a Slice. Slices are operated on in parallel, and they are evenly distributed across a cluster via a consistent hash algorithm.

212
executor_test.go
View file

@ -19,18 +19,18 @@ import (
"fmt"
"reflect"
"strconv"
"strings"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
// Ensure a bitmap query can be executed.
func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Run("Row", func(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true})
@ -38,10 +38,10 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set bits.
if _, err := e.Execute(context.Background(), "i", MustParse(``+
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1),
@ -52,7 +52,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{3, SliceWidth + 1}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -62,17 +62,17 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
})
t.Run("Column", func(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := index.CreateFrame("f", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set bits.
if _, err := e.Execute(context.Background(), "i", MustParse(``+
if _, err := e.Execute(context.Background(), "i", test.MustParse(``+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, 3)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 10, SliceWidth+1)+
fmt.Sprintf("SetBit(frame=f, rowID=%d, columnID=%d)\n", 20, SliceWidth+1),
@ -83,7 +83,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
t.Fatal(err)
}
if res, err := e.Execute(context.Background(), "i", MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(fmt.Sprintf(`Bitmap(columnID=%d, frame=f)`, SliceWidth+1)), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{10, 20}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -95,7 +95,7 @@ func TestExecutor_Execute_Bitmap(t *testing.T) {
// Ensure a difference query can be executed.
func TestExecutor_Execute_Difference(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 2)
@ -103,8 +103,8 @@ func TestExecutor_Execute_Difference(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 4)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "i", MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 3}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -113,19 +113,19 @@ func TestExecutor_Execute_Difference(t *testing.T) {
// Ensure an empty difference query behaves properly.
func TestExecutor_Execute_Empty_Difference(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "i", MustParse(`Difference()`), nil, nil); err == nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Difference()`), nil, nil); err == nil {
t.Fatalf("Empty Difference query should give error, but got %v", res)
}
}
// Ensure an intersect query can be executed.
func TestExecutor_Execute_Intersect(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 1)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
@ -135,8 +135,8 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -145,18 +145,18 @@ func TestExecutor_Execute_Intersect(t *testing.T) {
// Ensure an empty intersect query behaves properly.
func TestExecutor_Execute_Empty_Intersect(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "i", MustParse(`Intersect()`), nil, nil); err == nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Intersect()`), nil, nil); err == nil {
t.Fatalf("Empty Intersect query should give error, but got %v", res)
}
}
// Ensure a union query can be executed.
func TestExecutor_Execute_Union(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
@ -165,8 +165,8 @@ func TestExecutor_Execute_Union(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(11, 2)
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 1).MustSetBits(11, SliceWidth+2)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "i", MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union(Bitmap(rowID=10), Bitmap(rowID=11))`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{0, 2, SliceWidth + 1, SliceWidth + 2}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -175,12 +175,12 @@ func TestExecutor_Execute_Union(t *testing.T) {
// Ensure an empty union query behaves properly.
func TestExecutor_Execute_Empty_Union(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "general", pilosa.ViewStandard, 0).MustSetBits(10, 0)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "i", MustParse(`Union()`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Union()`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -189,14 +189,14 @@ func TestExecutor_Execute_Empty_Union(t *testing.T) {
// Ensure a count query can be executed.
func TestExecutor_Execute_Count(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).MustSetBits(10, 3)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, SliceWidth+2)
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(3) {
t.Fatalf("unexpected n: %d", res[0])
@ -205,16 +205,16 @@ func TestExecutor_Execute_Count(t *testing.T) {
// Ensure a set query can be executed.
func TestExecutor_Execute_SetBit(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
f := hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0)
if n := f.Row(11).Count(); n != 0 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if !res[0].(bool) {
@ -225,7 +225,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
if n := f.Row(11).Count(); n != 1 {
t.Fatalf("unexpected bitmap count: %d", n)
}
if res, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=11, frame=f, columnID=1)`), nil, nil); err != nil {
t.Fatal(err)
} else {
if res[0].(bool) {
@ -236,7 +236,7 @@ func TestExecutor_Execute_SetBit(t *testing.T) {
// Ensure a SetRowAttrs() query can be executed.
func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
// Create frames.
@ -249,17 +249,17 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Set two fields on f/10.
// Also set fields on other bitmaps and frames to test isolation.
e := NewExecutor(hldr.Holder, NewCluster(1))
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=200, frame=f, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=xxx, YYY=1)`), nil, nil); err != nil {
t.Fatal(err)
}
if _, err := e.Execute(context.Background(), "i", MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetRowAttrs(rowID=10, frame=f, baz=123, bat=true)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -273,9 +273,9 @@ func TestExecutor_Execute_SetRowAttrs(t *testing.T) {
// Ensure a TopN() query can be executed.
func TestExecutor_Execute_TopN(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Set bits for rows 0, 10, & 20 across two slices.
if idx, err := hldr.CreateIndex("i", pilosa.IndexOptions{}); err != nil {
@ -284,7 +284,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
t.Fatal(err)
} else if _, err := idx.CreateFrame("other", pilosa.FrameOptions{InverseEnabled: true}); err != nil {
t.Fatal(err)
} else if _, err := e.Execute(context.Background(), "i", MustParse(`
} else if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=0, columnID=0)
SetBit(frame=f, rowID=0, columnID=1)
SetBit(frame=f, rowID=0, columnID=`+strconv.Itoa(SliceWidth)+`)
@ -304,7 +304,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 5).RecalculateCache()
t.Run("Standard", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{ID: 0, Count: 5},
@ -315,7 +315,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
})
t.Run("Inverse", func(t *testing.T) {
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil {
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, inverse=true, n=2)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result[0], []pilosa.Pair{
{ID: SliceWidth, Count: 3},
@ -326,7 +326,7 @@ func TestExecutor_Execute_TopN(t *testing.T) {
})
}
func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
@ -338,8 +338,8 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).SetBit(1, SliceWidth)
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 4},
@ -350,7 +350,7 @@ func TestExecutor_Execute_TopN_fill(t *testing.T) {
// Ensure
func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
@ -372,8 +372,8 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 3).SetBit(4, 3*SliceWidth+1)
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=1)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
@ -384,7 +384,7 @@ func TestExecutor_Execute_TopN_fill_small(t *testing.T) {
// Ensure a TopN() query with a source bitmap can be executed.
func TestExecutor_Execute_TopN_Src(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
// Set bits for rows 0, 10, & 20 across two slices.
@ -407,8 +407,8 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
hldr.MustCreateRankedFragmentIfNotExists("i", "other", pilosa.ViewStandard, 1).RecalculateCache()
// Execute query.
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=100, frame=other), frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 20, Count: 3},
@ -422,7 +422,7 @@ func TestExecutor_Execute_TopN_Src(t *testing.T) {
//Ensure TopN handles Attribute filters
func TestExecutor_Execute_TopN_Attr(t *testing.T) {
//
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
@ -431,8 +431,8 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": int64(123)}); err != nil {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -445,7 +445,7 @@ func TestExecutor_Execute_TopN_Attr(t *testing.T) {
//Ensure TopN handles Attribute filters with source bitmap
func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
//
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
@ -454,8 +454,8 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
if err := hldr.Frame("i", "f").RowAttrStore().SetAttrs(10, map[string]interface{}{"category": uint64(123)}); err != nil {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, NewCluster(1))
if result, err := e.Execute(context.Background(), "i", MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if result, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(Bitmap(rowID=10,frame=f),frame="f", n=1, field="category", filters=[123])`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(result, []interface{}{[]pilosa.Pair{
{ID: 10, Count: 1},
@ -467,9 +467,9 @@ func TestExecutor_Execute_TopN_Attr_Src(t *testing.T) {
// Ensure a range query can be executed.
func TestExecutor_Execute_Range(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
// Create index.
index := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
@ -483,7 +483,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
}
// Set bits.
if _, err := e.Execute(context.Background(), "i", MustParse(`
if _, err := e.Execute(context.Background(), "i", test.MustParse(`
SetBit(frame=f, rowID=1, columnID=2, timestamp="1999-12-31T00:00")
SetBit(frame=f, rowID=1, columnID=3, timestamp="2000-01-01T00:00")
SetBit(frame=f, rowID=1, columnID=4, timestamp="2000-01-02T00:00")
@ -499,7 +499,7 @@ func TestExecutor_Execute_Range(t *testing.T) {
}
t.Run("Standard", func(t *testing.T) {
if res, err := e.Execute(context.Background(), "i", MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(rowID=1, frame=f, start="1999-12-31T00:00", end="2002-01-01T03:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{2, 3, 4, 5, 6, 7}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -507,8 +507,8 @@ func TestExecutor_Execute_Range(t *testing.T) {
})
t.Run("Inverse", func(t *testing.T) {
e := NewExecutor(hldr.Holder, NewCluster(1))
if res, err := e.Execute(context.Background(), "i", MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Range(columnID=2, frame=f, start="1999-01-01T00:00", end="2003-01-01T00:00")`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 10}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -518,10 +518,10 @@ func TestExecutor_Execute_Range(t *testing.T) {
// Ensure a remote query can return a bitmap.
func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
c := NewCluster(2)
c := test.NewCluster(2)
// Create secondary server and update second cluster node.
s := NewServer()
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
@ -546,13 +546,13 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// Create local executor data.
// The local node owns slice 1.
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 1).MustSetBits(10, (1*SliceWidth)+1)
e := NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Bitmap(rowID=10, frame=f)`), nil, nil); err != nil {
t.Fatal(err)
} else if bits := res[0].(*pilosa.Bitmap).Bits(); !reflect.DeepEqual(bits, []uint64{1, 2, 2*SliceWidth + 4}) {
t.Fatalf("unexpected bits: %+v", bits)
@ -561,10 +561,10 @@ func TestExecutor_Execute_Remote_Bitmap(t *testing.T) {
// Ensure a remote query can return a count.
func TestExecutor_Execute_Remote_Count(t *testing.T) {
c := NewCluster(2)
c := test.NewCluster(2)
// Create secondary server and update second cluster node.
s := NewServer()
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
@ -574,14 +574,14 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
}
// Create local executor data. The local node owns slice 1.
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+1)
hldr.MustCreateFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(10, (2*SliceWidth)+2)
e := NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`Count(Bitmap(rowID=10, frame=f))`), nil, nil); err != nil {
t.Fatal(err)
} else if res[0] != uint64(12) {
t.Fatalf("unexpected n: %d", res[0])
@ -590,11 +590,11 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Ensure a remote query can set bits on multiple nodes.
func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
c := NewCluster(2)
c := test.NewCluster(2)
c.ReplicaN = 2
// Create secondary server and update second cluster node.
s := NewServer()
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
@ -611,7 +611,7 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
}
// Create local executor data.
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
@ -620,8 +620,8 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2)`), nil, nil); err != nil {
t.Fatal(err)
}
@ -636,11 +636,11 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Ensure a remote query can set bits on multiple nodes.
func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
c := NewCluster(2)
c := test.NewCluster(2)
c.ReplicaN = 2
// Create secondary server and update second cluster node.
s := NewServer()
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
@ -657,7 +657,7 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
}
// Create local executor data.
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
@ -668,8 +668,8 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
t.Fatal(err)
}
e := NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, c)
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit(rowID=10, frame=f, columnID=2, timestamp="2016-12-11T10:09")`), nil, nil); err != nil {
t.Fatal(err)
}
@ -684,10 +684,10 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Ensure a remote query can return a top-n query.
func TestExecutor_Execute_Remote_TopN(t *testing.T) {
c := NewCluster(2)
c := test.NewCluster(2)
// Create secondary server and update second cluster node.
s := NewServer()
s := test.NewServer()
defer s.Close()
c.Nodes[1].Host = s.Host()
@ -725,14 +725,14 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
}
// Create local executor data on slice 2 & 4.
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s.Handler.Holder = hldr.Holder
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 2).MustSetBits(30, (2*SliceWidth)+1)
hldr.MustCreateRankedFragmentIfNotExists("i", "f", pilosa.ViewStandard, 4).MustSetBits(30, (4*SliceWidth)+2)
e := NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
e := test.NewExecutor(hldr.Holder, c)
if res, err := e.Execute(context.Background(), "i", test.MustParse(`TopN(frame=f, n=3)`), nil, nil); err != nil {
t.Fatal(err)
} else if !reflect.DeepEqual(res, []interface{}{[]pilosa.Pair{
{ID: 0, Count: 5},
@ -745,35 +745,11 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Ensure executor returns an error if too many writes are in a single request.
func TestExecutor_Execute_ErrMaxWritesPerRequest(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e.MaxWritesPerRequest = 3
if _, err := e.Execute(context.Background(), "i", MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites {
if _, err := e.Execute(context.Background(), "i", test.MustParse(`SetBit() ClearBit() SetBit() SetBit()`), nil, nil); err != pilosa.ErrTooManyWrites {
t.Fatalf("unexpected error: %s", err)
}
}
// Executor represents a test wrapper for pilosa.Executor.
type Executor struct {
*pilosa.Executor
}
// NewExecutor returns a new instance of Executor.
// The executor always matches the hostname of the first cluster node.
func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
e := &Executor{Executor: pilosa.NewExecutor()}
e.Holder = holder
e.Cluster = cluster
e.Host = cluster.Nodes[0].Host
return e
}
// MustParse parses s into a PQL query. Panic on error.
func MustParse(s string) *pql.Query {
q, err := pql.NewParser(strings.NewReader(s)).Parse()
if err != nil {
panic(err)
}
return q
}

2
fragment.go
View file

@ -265,7 +265,7 @@ func (f *Fragment) openCache() error {
// Unmarshal cache data.
var pb internal.Cache
if err := proto.Unmarshal(buf, &pb); err != nil {
log.Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err)
f.logger().Printf("error unmarshaling cache data, skipping: path=%s, err=%s", path, err)
return nil
}

163
fragment_test.go
View file

@ -17,14 +17,13 @@ package pilosa_test
import (
"bytes"
"flag"
"io/ioutil"
"math"
"os"
"reflect"
"testing"
"github.com/davecgh/go-spew/spew"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// Test flags
@ -37,7 +36,7 @@ const SliceWidth = pilosa.SliceWidth
// Ensure a fragment can set a bit and retrieve it.
func TestFragment_SetBit(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set bits on the fragment.
@ -68,7 +67,7 @@ func TestFragment_SetBit(t *testing.T) {
// Ensure a fragment can clear a set bit.
func TestFragment_ClearBit(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set and then clear bits on the fragment.
@ -95,7 +94,7 @@ func TestFragment_ClearBit(t *testing.T) {
// Ensure a fragment can snapshot correctly.
func TestFragment_Snapshot(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set and then clear bits on the fragment.
@ -124,7 +123,7 @@ func TestFragment_Snapshot(t *testing.T) {
// Ensure a fragment can iterate over all bits in order.
func TestFragment_ForEachBit(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set bits on the fragment.
@ -153,7 +152,7 @@ func TestFragment_ForEachBit(t *testing.T) {
// Ensure a fragment can return the top n results.
func TestFragment_Top(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
// Set bits on the rows 100, 101, & 102.
f.MustSetBits(100, 1, 3, 200)
@ -175,7 +174,7 @@ func TestFragment_Top(t *testing.T) {
// Ensure a fragment can filter rows when retrieving the top n rows.
func TestFragment_Top_Filter(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
// Set bits on the rows 100, 101, & 102.
@ -205,7 +204,7 @@ func TestFragment_Top_Filter(t *testing.T) {
// Ensure a fragment can return top rows that intersect with an input row.
func TestFragment_TopN_Intersect(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
// Create an intersecting input row.
@ -236,7 +235,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
t.Skip("short mode")
}
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
// Create an intersecting input row.
@ -274,7 +273,7 @@ func TestFragment_TopN_Intersect_Large(t *testing.T) {
// Ensure a fragment can return top rows when specified by ID.
func TestFragment_TopN_IDs(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
// Set bits on various rows.
@ -299,7 +298,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
cacheSize := uint32(3)
// Create Index.
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
// Create frame.
@ -322,9 +321,9 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
// Close the storage so we can re-open it without encountering a flock.
frag.Close()
f := &Fragment{
f := &test.Fragment{
Fragment: frag,
RowAttrStore: MustOpenAttrStore(),
RowAttrStore: test.MustOpenAttrStore(),
}
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
if err := f.Open(); err != nil {
@ -362,7 +361,7 @@ func TestFragment_TopN_CacheSize(t *testing.T) {
// Ensure fragment can return a checksum for its blocks.
func TestFragment_Checksum(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Retrieve checksum and set bits.
@ -381,7 +380,7 @@ func TestFragment_Checksum(t *testing.T) {
// Ensure fragment can return a checksum for a given block.
func TestFragment_Blocks(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Retrieve initial checksum.
@ -419,7 +418,7 @@ func TestFragment_Blocks(t *testing.T) {
// Ensure fragment returns an empty checksum if no data exists for a block.
func TestFragment_Blocks_Empty(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
// Set bits on a different block.
@ -437,7 +436,7 @@ func TestFragment_Blocks_Empty(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_LRUCache_Persistence(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU)
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeLRU)
defer f.Close()
// Set bits on the fragment.
@ -469,7 +468,7 @@ func TestFragment_LRUCache_Persistence(t *testing.T) {
// Ensure a fragment's cache can be persisted between restarts.
func TestFragment_RankCache_Persistence(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
// Create frame.
@ -522,7 +521,7 @@ func TestFragment_RankCache_Persistence(t *testing.T) {
// Ensure a fragment can be copied to another fragment.
func TestFragment_WriteTo_ReadFrom(t *testing.T) {
f0 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f0 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f0.Close()
// Set and then clear bits on the fragment.
@ -547,7 +546,7 @@ func TestFragment_WriteTo_ReadFrom(t *testing.T) {
}
// Read into another fragment.
f1 := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f1 := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
if rn, err := f1.ReadFrom(&buf); err != nil {
t.Fatal(err)
} else if wn != rn {
@ -596,7 +595,7 @@ func BenchmarkFragment_Blocks(b *testing.B) {
}
func BenchmarkFragment_IntersectionCount(b *testing.B) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, "")
defer f.Close()
f.MaxOpN = math.MaxInt32
@ -626,124 +625,8 @@ func BenchmarkFragment_IntersectionCount(b *testing.B) {
}
}
// Fragment is a test wrapper for pilosa.Fragment.
type Fragment struct {
*pilosa.Fragment
RowAttrStore *AttrStore
}
// NewFragment returns a new instance of Fragment with a temporary path.
func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fragment {
file, err := ioutil.TempFile("", "pilosa-fragment-")
if err != nil {
panic(err)
}
file.Close()
f := &Fragment{
Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice),
RowAttrStore: MustOpenAttrStore(),
}
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
return f
}
// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error.
func MustOpenFragment(index, frame, view string, slice uint64, cacheType string) *Fragment {
if cacheType == "" {
cacheType = pilosa.DefaultCacheType
}
f := NewFragment(index, frame, view, slice, cacheType)
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the fragment and removes all underlying data.
func (f *Fragment) Close() error {
defer os.Remove(f.Path())
defer os.Remove(f.CachePath())
defer f.RowAttrStore.Close()
return f.Fragment.Close()
}
// Reopen closes the fragment and reopens it as a new instance.
func (f *Fragment) Reopen() error {
cacheType := f.Fragment.CacheType
path := f.Path()
if err := f.Fragment.Close(); err != nil {
return err
}
f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice())
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
if err := f.Open(); err != nil {
return err
}
return nil
}
// MustSetBits sets bits on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.SetBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// MustClearBits clears bits on a row. Panic on error.
func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.ClearBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// RowAttrStore provides simple storage for attributes.
type RowAttrStore struct {
attrs map[uint64]map[string]interface{}
}
// NewRowAttrStore returns a new instance of RowAttrStore.
func NewRowAttrStore() *RowAttrStore {
return &RowAttrStore{
attrs: make(map[uint64]map[string]interface{}),
}
}
// RowAttrs returns the attributes set to a row id.
func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) {
return s.attrs[id], nil
}
// SetRowAttrs assigns a set of attributes to a row id.
func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) {
s.attrs[id] = m
}
// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk.
func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
ipct := int(pct * 100)
for i := 0; i < SliceWidth*rowN; i++ {
if i%100 >= ipct {
continue
}
rowIDs = append(rowIDs, uint64(i%SliceWidth))
columnIDs = append(columnIDs, uint64(i/SliceWidth))
}
return
}
func TestFragment_Tanimoto(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
src := pilosa.NewBitmap(1, 2, 3)
@ -766,7 +649,7 @@ func TestFragment_Tanimoto(t *testing.T) {
}
func TestFragment_Zero_Tanimoto(t *testing.T) {
f := MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
f := test.MustOpenFragment("i", "f", pilosa.ViewStandard, 0, pilosa.CacheTypeRanked)
defer f.Close()
src := pilosa.NewBitmap(1, 2, 3)

89
frame_test.go
View file

@ -16,16 +16,15 @@ package pilosa_test
import (
"io/ioutil"
"os"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// Ensure frame can open and retrieve a view.
func TestFrame_CreateViewIfNotExists(t *testing.T) {
f := MustOpenFrame()
f := test.MustOpenFrame()
defer f.Close()
// Create view.
@ -51,7 +50,7 @@ func TestFrame_CreateViewIfNotExists(t *testing.T) {
// Ensure frame can set its time quantum.
func TestFrame_SetTimeQuantum(t *testing.T) {
f := MustOpenFrame()
f := test.MustOpenFrame()
defer f.Close()
// Set & retrieve time quantum.
@ -161,85 +160,3 @@ func TestFrame_RowLabelValidation(t *testing.T) {
}
}
// Frame represents a test wrapper for pilosa.Frame.
type Frame struct {
*pilosa.Frame
}
// NewFrame returns a new instance of Frame d/0.
func NewFrame() *Frame {
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "i", "f")
if err != nil {
panic(err)
}
return &Frame{Frame: frame}
}
// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error.
func MustOpenFrame() *Frame {
f := NewFrame()
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the frame and removes the underlying data.
func (f *Frame) Close() error {
defer os.RemoveAll(f.Path())
return f.Frame.Close()
}
// Reopen closes the index and reopens it.
func (f *Frame) Reopen() error {
var err error
if err := f.Frame.Close(); err != nil {
return err
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Frame, err = pilosa.NewFrame(path, index, name)
if err != nil {
return err
}
if err := f.Open(); err != nil {
return err
}
return nil
}
// MustSetBit sets a bit on the frame. Panic on error.
func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) {
changed, err := f.SetBit(view, rowID, columnID, t)
if err != nil {
panic(err)
}
return changed
}
// Ensure frame can set its cache
func TestFrame_SetCacheSize(t *testing.T) {
f := MustOpenFrame()
defer f.Close()
cacheSize := uint32(100)
// Set & retrieve frame cache size.
if err := f.SetCacheSize(cacheSize); err != nil {
t.Fatal(err)
} else if q := f.CacheSize(); q != cacheSize {
t.Fatalf("unexpected frame cache size: %d", q)
}
// Reload frame and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.CacheSize(); q != cacheSize {
t.Fatalf("unexpected frame cache size (reopen): %d", q)
}
}

7
gossip/gossip.go
View file

@ -18,7 +18,6 @@ import (
"fmt"
"io"
"log"
"os"
"golang.org/x/sync/errgroup"
@ -98,9 +97,9 @@ type gossipConfig struct {
}
// NewGossipNodeSet returns a new instance of GossipNodeSet.
func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, sh pilosa.StatusHandler) *GossipNodeSet {
func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed string, server *pilosa.Server) *GossipNodeSet {
g := &GossipNodeSet{
LogOutput: os.Stderr,
LogOutput: server.LogOutput,
}
//TODO: pull memberlist config from pilosa.cfg file
@ -115,7 +114,7 @@ func NewGossipNodeSet(name string, gossipHost string, gossipPort int, gossipSeed
g.config.memberlistConfig.AdvertisePort = gossipPort
g.config.memberlistConfig.Delegate = g
g.statusHandler = sh
g.statusHandler = server
return g
}

369
handler_test.go
View file

@ -17,13 +17,10 @@ package pilosa_test
import (
"bytes"
"context"
"encoding/json"
"errors"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"reflect"
"strings"
"testing"
@ -32,19 +29,20 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_NotFound(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/no_such_path", nil))
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil))
if w.Code != http.StatusNotFound {
t.Fatalf("invalid status: %d", w.Code)
}
@ -52,7 +50,7 @@ func TestHandler_NotFound(t *testing.T) {
// Ensure the handler can return the schema.
func TestHandler_Schema(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
@ -74,11 +72,11 @@ func TestHandler_Schema(t *testing.T) {
t.Fatal(err)
}
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(1)
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/schema", nil))
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","frames":[{"name":"f0"},{"name":"f1","views":[{"name":"inverse"},{"name":"standard"}]}]},{"name":"i1","frames":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
@ -88,8 +86,8 @@ func TestHandler_Schema(t *testing.T) {
// Ensure the handler can return the status.
func TestHandler_Status(t *testing.T) {
s := NewServer()
hldr := MustOpenHolder()
s := test.NewServer()
hldr := test.MustOpenHolder()
defer s.Close()
defer hldr.Close()
@ -112,14 +110,14 @@ func TestHandler_Status(t *testing.T) {
t.Fatal(err)
}
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(1)
h.Cluster = test.NewCluster(1)
h.StatusHandler = s
s.Handler = h
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/status", nil))
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"status":{"State":"UP","Indexes":[{"Name":"i0","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}},{"Name":"f1","Meta":{"RowLabel":"rowID","InverseEnabled":true,"CacheType":"ranked","CacheSize":50000}}]},{"Name":"i1","Meta":{"ColumnLabel":"columnID"},"Frames":[{"Name":"f0","Meta":{"RowLabel":"rowID","CacheType":"ranked","CacheSize":50000}}]}]}}`+"\n" {
@ -129,7 +127,7 @@ func TestHandler_Status(t *testing.T) {
// Ensure the handler can return the maxslice map.
func TestHandler_MaxSlices(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("i0", "f0", pilosa.ViewStandard, 1).MustSetBits(30, (1*SliceWidth)+1)
@ -140,11 +138,11 @@ func TestHandler_MaxSlices(t *testing.T) {
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+2)
hldr.MustCreateFragmentIfNotExists("i1", "f1", pilosa.ViewStandard, 0).MustSetBits(40, (0*SliceWidth)+8)
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(1)
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max", nil))
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
@ -154,7 +152,7 @@ func TestHandler_MaxSlices(t *testing.T) {
// Ensure the handler can return the maxslice map for the inverse views.
func TestHandler_MaxSlices_Inverse(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
f0, err := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{}).CreateFrame("f0", pilosa.FrameOptions{InverseEnabled: true})
@ -181,11 +179,11 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
t.Fatal(err)
}
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(1)
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max?inverse=true", nil))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"maxSlices":{"i0":3,"i1":0}}`+"\n" {
@ -195,11 +193,11 @@ func TestHandler_MaxSlices_Inverse(t *testing.T) {
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
@ -213,7 +211,7 @@ func TestHandler_Query_Args_URL(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
@ -223,11 +221,11 @@ func TestHandler_Query_Args_URL(t *testing.T) {
// Ensure the handler can accept arguments via protobufs.
func TestHandler_Query_Args_Protobuf(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
@ -250,7 +248,7 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
}
// Generate protobuf request.
req := MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody))
req := test.MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/x-protobuf")
w := httptest.NewRecorder()
@ -263,14 +261,14 @@ func TestHandler_Query_Args_Protobuf(t *testing.T) {
// Ensure the handler returns an error when parsing bad arguments.
func TestHandler_Query_Args_Err(t *testing.T) {
w := httptest.NewRecorder()
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
@ -279,7 +277,7 @@ func TestHandler_Query_Args_Err(t *testing.T) {
}
func TestHandler_Query_Params_Err(t *testing.T) {
w := httptest.NewRecorder()
NewHandler().ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)")))
test.NewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid query params"}`+"\n" {
@ -290,18 +288,18 @@ func TestHandler_Query_Params_Err(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as JSON.
func TestHandler_Query_Uint64_JSON(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Bitmap( id=100))")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
@ -311,18 +309,18 @@ func TestHandler_Query_Uint64_JSON(t *testing.T) {
// Ensure the handler can execute a query with a uint64 response as protobufs.
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))"))
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Bitmap(id=100))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
@ -339,11 +337,11 @@ func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap as JSON.
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
@ -352,7 +350,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}]}`+"\n" {
@ -362,7 +360,7 @@ func TestHandler_Query_Bitmap_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap with column attributes as JSON.
func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
hldr := NewHolder()
hldr := test.NewHolder()
defer hldr.Close()
// Create index and set column attributes.
@ -375,9 +373,9 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
t.Fatal(err)
}
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(1)
h.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, 3, 66, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
@ -385,7 +383,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Bitmap(id=100)")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"bits":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
@ -395,11 +393,11 @@ func TestHandler_Query_Bitmap_ColumnAttrs_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap as protobuf.
func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
@ -408,7 +406,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Bitmap(id=100)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
@ -433,7 +431,7 @@ func TestHandler_Query_Bitmap_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns a bitmap with column attributes as protobuf.
func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
hldr := NewHolder()
hldr := test.NewHolder()
defer hldr.Close()
// Create index and set column attributes.
@ -444,9 +442,9 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
t.Fatal(err)
}
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(1)
h.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
bm := pilosa.NewBitmap(1, pilosa.SliceWidth+1)
bm.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
@ -463,7 +461,7 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf))
r := test.MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf))
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
@ -500,11 +498,11 @@ func TestHandler_Query_Bitmap_ColumnAttrs_Protobuf(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as JSON.
func TestHandler_Query_Pairs_JSON(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
@ -514,7 +512,7 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" {
@ -524,11 +522,11 @@ func TestHandler_Query_Pairs_JSON(t *testing.T) {
// Ensure the handler can execute a query that returns pairs as protobuf.
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
@ -538,7 +536,7 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
@ -555,18 +553,18 @@ func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
// Ensure the handler can return an error as JSON.
func TestHandler_Query_Err_JSON(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`)))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Bitmap(id=100)`)))
if w.Code != http.StatusInternalServerError {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"marker"}`+"\n" {
@ -576,18 +574,18 @@ func TestHandler_Query_Err_JSON(t *testing.T) {
// Ensure the handler can return an error as protobuf.
func TestHandler_Query_Err_Protobuf(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
r := MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(frame=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != http.StatusInternalServerError {
@ -604,14 +602,14 @@ func TestHandler_Query_Err_Protobuf(t *testing.T) {
// Ensure the handler returns "method not allowed" for non-POST queries.
func TestHandler_Query_MethodNotAllowed(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("GET", "/index/i/query", nil))
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil))
if w.Code != http.StatusMethodNotAllowed {
t.Fatalf("invalid status: %d", w.Code)
}
@ -619,14 +617,14 @@ func TestHandler_Query_MethodNotAllowed(t *testing.T) {
// Ensure the handler returns an error if there is a parsing error..
func TestHandler_Query_ErrParse(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != http.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"expected comma, right paren, or identifier, found \"\" occurred at line 1, char 8"}`+"\n" {
@ -636,10 +634,10 @@ func TestHandler_Query_ErrParse(t *testing.T) {
// Ensure the handler can delete an index.
func TestHandler_Index_Delete(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
defer s.Close()
@ -649,7 +647,7 @@ func TestHandler_Index_Delete(t *testing.T) {
}
// Send request to delete index.
resp, err := http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
resp, err := http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
if err != nil {
t.Fatal(err)
}
@ -672,18 +670,18 @@ func TestHandler_Index_Delete(t *testing.T) {
// Ensure handler can delete a frame.
func TestHandler_DeleteFrame(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
if _, err := i0.CreateFrameIfNotExists("f1", pilosa.FrameOptions{}); err != nil {
t.Fatal(err)
}
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(1)
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader("")))
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/frame/f1", strings.NewReader("")))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
@ -695,15 +693,15 @@ func TestHandler_DeleteFrame(t *testing.T) {
// Ensure handler can set the Index time quantum.
func TestHandler_SetIndexTimeQuantum(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(1)
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
@ -715,7 +713,7 @@ func TestHandler_SetIndexTimeQuantum(t *testing.T) {
// Ensure handler can set the frame time quantum.
func TestHandler_SetFrameTimeQuantum(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
// Create frame.
@ -723,11 +721,11 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) {
t.Fatal(err)
}
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(1)
h.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
h.ServeHTTP(w, test.MustNewHTTPRequest("PATCH", "/index/i0/frame/f1/time-quantum", strings.NewReader(`{"timeQuantum":"ymdh"}`)))
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
@ -739,10 +737,10 @@ func TestHandler_SetFrameTimeQuantum(t *testing.T) {
// Ensure the handler can return data in differing blocks for an index.
func TestHandler_Index_AttrStore_Diff(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
defer s.Close()
@ -773,7 +771,7 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) {
resp, err := http.Post(
s.URL+"/index/i/attr/diff",
"application/json",
strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`),
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
if err != nil {
t.Fatal(err)
@ -781,17 +779,17 @@ func TestHandler_Index_AttrStore_Diff(t *testing.T) {
defer resp.Body.Close()
// Read and validate body.
if body := string(MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can return data in differing blocks for a frame.
func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
defer s.Close()
@ -823,7 +821,7 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
resp, err := http.Post(
s.URL+"/index/i/frame/meta/attr/diff",
"application/json",
strings.NewReader(`{"blocks":`+string(MustMarshalJSON(blks))+`}`),
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
if err != nil {
t.Fatal(err)
@ -831,17 +829,17 @@ func TestHandler_Frame_AttrStore_Diff(t *testing.T) {
defer resp.Body.Close()
// Read and validate body.
if body := string(MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can backup a fragment and then restore it.
func TestHandler_Fragment_BackupRestore(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
defer s.Close()
@ -887,15 +885,15 @@ func TestHandler_Fragment_BackupRestore(t *testing.T) {
// Ensure the handler can retrieve the version.
func TestHandler_Version(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
w := httptest.NewRecorder()
r := MustNewHTTPRequest("GET", "/version", nil)
r := test.MustNewHTTPRequest("GET", "/version", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
@ -906,16 +904,16 @@ func TestHandler_Version(t *testing.T) {
// Ensure the handler can return a list of nodes for a fragment.
func TestHandler_Fragment_Nodes(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h := test.NewHandler()
h.Holder = hldr.Holder
h.Cluster = NewCluster(3)
h.Cluster = test.NewCluster(3)
h.Cluster.ReplicaN = 2
w := httptest.NewRecorder()
r := MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil)
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
@ -926,147 +924,20 @@ func TestHandler_Fragment_Nodes(t *testing.T) {
// Ensure the handler can return expvars without panicking.
func TestHandler_Expvars(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := NewHandler()
h.Cluster = NewCluster(1)
h := test.NewHandler()
h.Cluster = test.NewCluster(1)
h.Holder = hldr.Holder
w := httptest.NewRecorder()
r := MustNewHTTPRequest("GET", "/debug/vars", nil)
r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
if w.Code != http.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Handler represents a test wrapper for pilosa.Handler.
type Handler struct {
*pilosa.Handler
Executor HandlerExecutor
}
// NewHandler returns a new instance of Handler.
func NewHandler() *Handler {
h := &Handler{
Handler: pilosa.NewHandler(),
}
h.Handler.Executor = &h.Executor
h.Handler.LogOutput = ioutil.Discard
// Handler test messages can no-op.
h.Broadcaster = pilosa.NopBroadcaster
return h
}
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return c.ExecuteFn(ctx, index, query, slices, opt)
}
// Server represents a test wrapper for httptest.Server.
type Server struct {
*httptest.Server
Handler *Handler
}
// NewServer returns a test server running on a random port.
func NewServer() *Server {
s := &Server{
Handler: NewHandler(),
}
s.Server = httptest.NewServer(s.Handler.Handler)
// Update handler to use hostname.
s.Handler.Host = s.Host()
// Handler test messages can no-op.
s.Handler.Broadcaster = pilosa.NopBroadcaster
// Create a default cluster on the handler
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
return s
}
// LocalStatus returns the state of the local node as well as the
// holder (indexes/frames) according to the local node.
func (s *Server) LocalStatus() (proto.Message, error) {
if s.Handler.Holder == nil {
return nil, errors.New("Server.Holder is nil")
}
ns := internal.NodeStatus{
Host: s.Handler.Handler.Host,
State: pilosa.NodeStateUp,
Indexes: pilosa.EncodeIndexes(s.Handler.Holder.Indexes()),
}
// Append Slice list per this Node's indexes
for _, index := range ns.Indexes {
index.Slices = s.Handler.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.Host)
}
return &ns, nil
}
// ClusterStatus returns the NodeState for all nodes in the cluster.
func (s *Server) ClusterStatus() (proto.Message, error) {
// Assuming we are only testing this with one Node
// So just return its status
return s.LocalStatus()
}
// HandleRemoteStatus just need to implement a nop to complete the Interface
func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil }
// Host returns the hostname of the running server.
func (s *Server) Host() string { return MustParseURLHost(s.URL) }
// MustParseURLHost parses rawurl and returns the hostname. Panic on error.
func MustParseURLHost(rawurl string) string {
u, err := url.Parse(rawurl)
if err != nil {
panic(err)
}
return u.Host
}
// MustNewHTTPRequest creates a new HTTP request. Panic on error.
func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request {
req, err := http.NewRequest(method, urlStr, body)
if err != nil {
panic(err)
}
return req
}
// MustMarshalJSON marshals v to JSON. Panic on error.
func MustMarshalJSON(v interface{}) []byte {
buf, err := json.Marshal(v)
if err != nil {
panic(err)
}
return buf
}
// MustReadAll reads a reader into a buffer and returns it. Panic on error.
func MustReadAll(r io.Reader) []byte {
buf, err := ioutil.ReadAll(r)
if err != nil {
panic(err)
}
return buf
}
// Ensure handler can create a input definition.
func TestHandler_CreateInputDefinition(t *testing.T) {
hldr := MustOpenHolder()

149
holder_test.go
View file

@ -15,9 +15,7 @@
package pilosa_test
import (
"bytes"
"context"
"io/ioutil"
"os"
"path/filepath"
"reflect"
@ -26,11 +24,12 @@ import (
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
func TestHolder_Open(t *testing.T) {
t.Run("ErrIndexName", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if err := os.Mkdir(h.IndexPath("!"), 0777); err != nil {
@ -45,7 +44,7 @@ func TestHolder_Open(t *testing.T) {
})
t.Run("ErrIndexPermission", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
@ -60,7 +59,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrIndexMetaCorrupt", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
@ -74,7 +73,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrIndexAttrStoreCorrupt", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if _, err := h.CreateIndex("test", pilosa.IndexOptions{}); err != nil {
@ -89,7 +88,7 @@ func TestHolder_Open(t *testing.T) {
})
t.Run("ErrFramePermission", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@ -106,7 +105,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFrameMetaCorrupt", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@ -122,7 +121,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFrameAttrStoreCorrupt", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@ -139,7 +138,7 @@ func TestHolder_Open(t *testing.T) {
})
t.Run("ErrViewPermission", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@ -158,7 +157,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrViewFragmentsMkdir", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@ -178,7 +177,7 @@ func TestHolder_Open(t *testing.T) {
})
t.Run("ErrFragmentStoragePermission", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@ -199,7 +198,7 @@ func TestHolder_Open(t *testing.T) {
}
})
t.Run("ErrFragmentStorageCorrupt", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@ -220,7 +219,7 @@ func TestHolder_Open(t *testing.T) {
})
t.Run("ErrFragmentCachePermission", func(t *testing.T) {
h := MustOpenHolder()
h := test.MustOpenHolder()
defer h.Close()
if idx, err := h.CreateIndex("foo", pilosa.IndexOptions{}); err != nil {
@ -246,7 +245,7 @@ func TestHolder_Open(t *testing.T) {
// Ensure holder can delete an index and its underlying files.
func TestHolder_DeleteIndex(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
// Write bits to separate indexes.
@ -279,16 +278,16 @@ func TestHolder_DeleteIndex(t *testing.T) {
// Ensure holder can sync with a remote holder.
func TestHolderSyncer_SyncHolder(t *testing.T) {
cluster := NewCluster(2)
cluster := test.NewCluster(2)
// Create a local holder.
hldr0 := MustOpenHolder()
hldr0 := test.MustOpenHolder()
defer hldr0.Close()
// Create a remote holder wrapped by an HTTP
hldr1 := MustOpenHolder()
hldr1 := test.MustOpenHolder()
defer hldr1.Close()
s := NewServer()
s := test.NewServer()
defer s.Close()
s.Handler.Holder = hldr1.Holder
s.Handler.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
@ -302,10 +301,10 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
// Mock 2-node, fully replicated cluster.
cluster.ReplicaN = 2
cluster.Nodes[0].Host = "localhost:0"
cluster.Nodes[1].Host = MustParseURLHost(s.URL)
cluster.Nodes[1].Host = test.MustParseURLHost(s.URL)
// Create frames on nodes.
for _, hldr := range []*Holder{hldr0, hldr1} {
for _, hldr := range []*test.Holder{hldr0, hldr1} {
hldr.MustCreateFrameIfNotExists("i", "f")
hldr.MustCreateFrameIfNotExists("i", "f0")
hldr.MustCreateFrameIfNotExists("y", "z")
@ -365,7 +364,7 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
}
// Verify data is the same on both nodes.
for i, hldr := range []*Holder{hldr0, hldr1} {
for i, hldr := range []*test.Holder{hldr0, hldr1} {
f := hldr.Fragment("i", "f", pilosa.ViewStandard, 0)
if a := f.Row(0).Bits(); !reflect.DeepEqual(a, []uint64{10, 4000}) {
t.Fatalf("unexpected bits(%d/0): %+v", i, a)
@ -393,109 +392,3 @@ func TestHolderSyncer_SyncHolder(t *testing.T) {
}
}
}
// Holder is a test wrapper for pilosa.Holder.
type Holder struct {
*pilosa.Holder
LogOutput bytes.Buffer
}
// NewHolder returns a new instance of Holder with a temporary path.
func NewHolder() *Holder {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
h := &Holder{Holder: pilosa.NewHolder()}
h.Path = path
h.Holder.LogOutput = &h.LogOutput
return h
}
// MustOpenHolder creates and opens a holder at a temporary path. Panic on error.
func MustOpenHolder() *Holder {
h := NewHolder()
if err := h.Open(); err != nil {
panic(err)
}
return h
}
// Close closes the holder and removes all underlying data.
func (h *Holder) Close() error {
defer os.RemoveAll(h.Path)
return h.Holder.Close()
}
// Reopen closes the holder and instantiates and opens a new holder.
func (h *Holder) Reopen() error {
if err := h.Holder.Close(); err != nil {
return err
}
path, logOutput := h.Path, h.Holder.LogOutput
h.Holder = pilosa.NewHolder()
h.Holder.Path = path
h.Holder.LogOutput = logOutput
if err := h.Holder.Open(); err != nil {
return err
}
return nil
}
// MustCreateIndexIfNotExists returns a given index. Panic on error.
func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index {
idx, err := h.Holder.CreateIndexIfNotExists(index, opt)
if err != nil {
panic(err)
}
return &Index{Index: idx}
}
// MustCreateFrameIfNotExists returns a given frame. Panic on error.
func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame {
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
if err != nil {
panic(err)
}
return f
}
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
if err != nil {
panic(err)
}
v, err := f.CreateViewIfNotExists(view)
if err != nil {
panic(err)
}
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
panic(err)
}
return &Fragment{Fragment: frag}
}
// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
if err != nil {
panic(err)
}
v, err := f.CreateViewIfNotExists(view)
if err != nil {
panic(err)
}
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
panic(err)
}
return &Fragment{Fragment: frag}
}

102
index_test.go
View file

@ -15,19 +15,18 @@
package pilosa_test
import (
"io/ioutil"
"os"
"reflect"
"strings"
"testing"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/test"
)
// Ensure index can open and retrieve a frame.
func TestIndex_CreateFrameIfNotExists(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
// Create frame.
@ -55,7 +54,7 @@ func TestIndex_CreateFrame(t *testing.T) {
// Ensure time quantum can be set appropriately on a new frame.
t.Run("TimeQuantum", func(t *testing.T) {
t.Run("Explicit", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
// Set index time quantum.
@ -73,7 +72,7 @@ func TestIndex_CreateFrame(t *testing.T) {
})
t.Run("Inherited", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
// Set index time quantum.
@ -94,7 +93,7 @@ func TestIndex_CreateFrame(t *testing.T) {
// Ensure frame can include range columns.
t.Run("RangeEnabled", func(t *testing.T) {
t.Run("OK", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
// Create frame with schema and verify it exists.
@ -129,7 +128,7 @@ func TestIndex_CreateFrame(t *testing.T) {
})
t.Run("ErrInverseRangeNotAllowed", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
@ -141,7 +140,7 @@ func TestIndex_CreateFrame(t *testing.T) {
})
t.Run("ErrRangeCacheNotAllowed", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
@ -153,7 +152,7 @@ func TestIndex_CreateFrame(t *testing.T) {
})
t.Run("ErrFrameFieldsNotAllowed", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
@ -166,7 +165,7 @@ func TestIndex_CreateFrame(t *testing.T) {
})
t.Run("ErrFieldNameRequired", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
@ -180,7 +179,7 @@ func TestIndex_CreateFrame(t *testing.T) {
})
t.Run("ErrInvalidFieldType", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
@ -194,7 +193,7 @@ func TestIndex_CreateFrame(t *testing.T) {
})
t.Run("ErrInvalidFieldRange", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
if _, err := index.CreateFrame("f", pilosa.FrameOptions{
@ -211,7 +210,7 @@ func TestIndex_CreateFrame(t *testing.T) {
// Ensure frame cannot be created with a matching row label.
t.Run("ErrColumnRowLabelEqual", func(t *testing.T) {
t.Run("Explicit", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
_, err := index.CreateFrame("f", pilosa.FrameOptions{RowLabel: pilosa.DefaultColumnLabel})
@ -221,7 +220,7 @@ func TestIndex_CreateFrame(t *testing.T) {
})
t.Run("Default", func(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
if err := index.SetColumnLabel(pilosa.DefaultRowLabel); err != nil {
t.Fatal(err)
@ -237,7 +236,7 @@ func TestIndex_CreateFrame(t *testing.T) {
// Ensure index can delete a frame.
func TestIndex_DeleteFrame(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
// Create frame.
@ -260,7 +259,7 @@ func TestIndex_DeleteFrame(t *testing.T) {
// Ensure index can set the default time quantum.
func TestIndex_SetTimeQuantum(t *testing.T) {
index := MustOpenIndex()
index := test.MustOpenIndex()
defer index.Close()
// Set & retrieve time quantum.
@ -277,77 +276,6 @@ func TestIndex_SetTimeQuantum(t *testing.T) {
t.Fatalf("unexpected quantum (reopen): %s", q)
}
}
// Index represents a test wrapper for pilosa.Index.
type Index struct {
*pilosa.Index
}
// NewIndex returns a new instance of Index.
func NewIndex() *Index {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := pilosa.NewIndex(path, "i")
if err != nil {
panic(err)
}
return &Index{Index: index}
}
// MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func MustOpenIndex() *Index {
index := NewIndex()
if err := index.Open(); err != nil {
panic(err)
}
return index
}
// Close closes the index and removes the underlying data.
func (i *Index) Close() error {
defer os.RemoveAll(i.Path())
return i.Index.Close()
}
// Reopen closes the index and reopens it.
func (i *Index) Reopen() error {
var err error
if err := i.Index.Close(); err != nil {
return err
}
path, name := i.Path(), i.Name()
i.Index, err = pilosa.NewIndex(path, name)
if err != nil {
return err
}
if err := i.Open(); err != nil {
return err
}
return nil
}
// CreateFrame creates a frame with the given options.
func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrame(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrameIfNotExists(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
}
// Ensure index can delete a frame.
func TestIndex_InvalidName(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-index-")

16
server.go
View file

@ -129,6 +129,7 @@ func (s *Server) Open() error {
}
// Open holder.
s.Holder.LogOutput = s.LogOutput
if err := s.Holder.Open(); err != nil {
return fmt.Errorf("opening Holder: %v", err)
}
@ -159,7 +160,6 @@ func (s *Server) Open() error {
// Initialize Holder.
s.Holder.Broadcaster = s.Broadcaster
s.Holder.LogOutput = s.LogOutput
// Serve HTTP.
go func() { http.Serve(ln, s.Handler) }()
@ -197,14 +197,14 @@ func (s *Server) Addr() net.Addr {
return s.ln.Addr()
}
func (s *Server) logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) }
func (s *Server) Logger() *log.Logger { return log.New(s.LogOutput, "", log.LstdFlags) }
func (s *Server) monitorAntiEntropy() {
t := time.Now()
ticker := time.NewTicker(s.AntiEntropyInterval)
defer ticker.Stop()
s.logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval)
s.Logger().Printf("holder sync monitor initializing (%s interval)", s.AntiEntropyInterval)
for {
// Wait for tick or a close.
@ -215,7 +215,7 @@ func (s *Server) monitorAntiEntropy() {
s.Holder.Stats.Count("AntiEntropy", 1, 1.0)
}
s.logger().Printf("holder sync beginning")
s.Logger().Printf("holder sync beginning")
// Initialize syncer with local holder and remote client.
var syncer HolderSyncer
@ -226,12 +226,12 @@ func (s *Server) monitorAntiEntropy() {
// Sync holders.
if err := syncer.SyncHolder(); err != nil {
s.logger().Printf("holder sync error: err=%s", err)
s.Logger().Printf("holder sync error: err=%s", err)
continue
}
// Record successful sync in log.
s.logger().Printf("holder sync complete")
s.Logger().Printf("holder sync complete")
}
dif := time.Since(t)
s.Holder.Stats.Histogram("AntiEntropyDuration", float64(dif), 1.0)
@ -267,7 +267,7 @@ func (s *Server) monitorMaxSlices() {
localIndex.SetRemoteMaxSlice(newmax)
}
} else {
s.logger().Printf("Local Index not found: %s", index)
s.Logger().Printf("Local Index not found: %s", index)
}
}
}
@ -484,7 +484,7 @@ func (s *Server) monitorRuntime() {
gcn := gcnotifier.New()
defer gcn.Close()
s.logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval)
s.Logger().Printf("runtime stats initializing (%s interval)", s.MetricInterval)
for {
// Wait for tick or a close.

32
server/server.go
View file

@ -100,7 +100,8 @@ func (m *Command) Run(args ...string) (err error) {
if err = m.Server.Open(); err != nil {
return fmt.Errorf("server.Open: %v", err)
}
fmt.Fprintf(m.Stderr, "Listening as http://%s\n", m.Server.Host)
m.Server.Logger().Printf("Listening as http://%s\n", m.Server.Host)
return nil
}
@ -122,18 +123,13 @@ func (m *Command) SetupServer() error {
m.Server.Cluster = cluster
// Setup logging output.
if m.Config.LogPath == "" {
m.Server.LogOutput = m.Stderr
} else {
logFile, err := os.OpenFile(m.Config.LogPath, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return err
}
m.Server.LogOutput = logFile
m.Server.LogOutput, err = GetLogWriter(m.Config.LogPath, m.Stderr)
if err != nil {
return err
}
// Configure holder.
fmt.Fprintf(m.Stderr, "Using data from: %s\n", m.Config.DataDir)
m.Server.Logger().Printf("Using data from: %s\n", m.Config.DataDir)
m.Server.Holder.Path = m.Config.DataDir
m.Server.MetricInterval = time.Duration(m.Config.Metric.PollingInterval)
m.Server.Holder.Stats, err = NewStatsClient(m.Config.Metric.Service, m.Config.Metric.Host)
@ -160,7 +156,7 @@ func (m *Command) SetupServer() error {
switch m.Config.Cluster.Type {
case "http":
m.Server.Broadcaster = httpbroadcast.NewHTTPBroadcaster(m.Server, internalPortStr)
m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(internalPortStr, m.Stderr)
m.Server.BroadcastReceiver = httpbroadcast.NewHTTPBroadcastReceiver(internalPortStr, m.Server.LogOutput)
m.Server.Cluster.NodeSet = httpbroadcast.NewHTTPNodeSet()
err := m.Server.Cluster.NodeSet.(*httpbroadcast.HTTPNodeSet).Join(m.Server.Cluster.Nodes)
if err != nil {
@ -202,6 +198,20 @@ func (m *Command) SetupServer() error {
return nil
}
// GetLogWriter opens a file for logging, or a default io.Writer (such as stderr) for an empty path.
func GetLogWriter(path string, defaultWriter io.Writer) (io.Writer, error) {
// This is split out so it can be used in NewServeCmd as well as SetupServer
if path == "" {
return defaultWriter, nil
} else {
logFile, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0600)
if err != nil {
return nil, err
}
return logFile, nil
}
}
func normalizeHost(host string) (string, error) {
if !strings.Contains(host, ":") {
host = host + ":"

51
stats_test.go
View file

@ -10,12 +10,13 @@ import (
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// TestMultiStatClient_Expvar run the multistat client with exp var
// since the EXPVAR data is stored in a global we should run these in one test function
func TestMultiStatClient_Expvar(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
c := pilosa.NewExpvarStatsClient()
@ -73,7 +74,7 @@ func TestMultiStatClient_Expvar(t *testing.T) {
}
func TestStatsCount_TopN(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
@ -83,7 +84,7 @@ func TestStatsCount_TopN(t *testing.T) {
// Execute query.
called := false
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "TopN" {
@ -98,7 +99,7 @@ func TestStatsCount_TopN(t *testing.T) {
return
},
}
if _, err := e.Execute(context.Background(), "d", MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`TopN(frame=f, n=2)`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -107,13 +108,13 @@ func TestStatsCount_TopN(t *testing.T) {
}
func TestStatsCount_Bitmap(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(0, 1)
called := false
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
e.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, tags []string) {
if name != "Bitmap" {
@ -128,7 +129,7 @@ func TestStatsCount_Bitmap(t *testing.T) {
return
},
}
if _, err := e.Execute(context.Background(), "d", MustParse(`Bitmap(frame=f, rowID=0)`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`Bitmap(frame=f, rowID=0)`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -137,14 +138,14 @@ func TestStatsCount_Bitmap(t *testing.T) {
}
func TestStatsCount_SetBitmapAttrs(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1)
called := false
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
frame := e.Holder.Frame("d", "f")
if frame == nil {
t.Fatal("frame not found")
@ -159,7 +160,7 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) {
return
},
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetRowAttrs(rowID=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -168,14 +169,14 @@ func TestStatsCount_SetBitmapAttrs(t *testing.T) {
}
func TestStatsCount_SetProfileAttrs(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 0)
hldr.MustCreateFragmentIfNotExists("d", "f", pilosa.ViewStandard, 0).SetBit(10, 1)
called := false
e := NewExecutor(hldr.Holder, NewCluster(1))
e := test.NewExecutor(hldr.Holder, test.NewCluster(1))
idx := e.Holder.Index("d")
if idx == nil {
t.Fatal("idex not found")
@ -191,7 +192,7 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
return
},
}
if _, err := e.Execute(context.Background(), "d", MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
if _, err := e.Execute(context.Background(), "d", test.MustParse(`SetColumnAttrs(id=10, frame=f, foo="bar")`), nil, nil); err != nil {
t.Fatal(err)
}
if !called {
@ -200,9 +201,9 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
}
func TestStatsCount_CreateIndex(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
defer s.Close()
called := false
@ -216,17 +217,17 @@ func TestStatsCount_CreateIndex(t *testing.T) {
return
},
}
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", nil))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", nil))
if !called {
t.Error("Count isn't called")
}
}
func TestStatsCount_DeleteIndex(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
defer s.Close()
@ -245,17 +246,17 @@ func TestStatsCount_DeleteIndex(t *testing.T) {
return
},
}
http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
}
func TestStatsCount_CreateFrame(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
defer s.Close()
@ -277,17 +278,17 @@ func TestStatsCount_CreateFrame(t *testing.T) {
return
},
}
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/frame/f", nil))
if !called {
t.Error("Count isn't called")
}
}
func TestStatsCount_DeleteFrame(t *testing.T) {
hldr := MustOpenHolder()
hldr := test.MustOpenHolder()
defer hldr.Close()
s := NewServer()
s := test.NewServer()
s.Handler.Holder = hldr.Holder
defer s.Close()
called := false
@ -309,7 +310,7 @@ func TestStatsCount_DeleteFrame(t *testing.T) {
return
},
}
http.DefaultClient.Do(MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/frame/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}

75
test/attr.go Normal file
View file

@ -0,0 +1,75 @@
package test
import (
"io/ioutil"
"os"
"runtime"
"sync"
"testing"
"github.com/pilosa/pilosa"
)
// AttrStore represents a test wrapper for pilosa.AttrStore.
type AttrStore struct {
*pilosa.AttrStore
}
// NewAttrStore returns a new instance of AttrStore.
func NewAttrStore() *AttrStore {
f, err := ioutil.TempFile("", "pilosa-attr-")
if err != nil {
panic(err)
}
f.Close()
os.Remove(f.Name())
return &AttrStore{AttrStore: pilosa.NewAttrStore(f.Name())}
}
func BenchmarkAttrStore_Duplicate(b *testing.B) {
s := MustOpenAttrStore()
defer s.Close()
// Set attributes.
const n = 5
for i := 0; i < n; i++ {
if err := s.SetAttrs(uint64(i), map[string]interface{}{"A": 100, "B": "foo", "C": true, "D": 100.2}); err != nil {
b.Fatal(err)
}
}
b.ReportAllocs()
b.ResetTimer()
// Update attributes with an existing subset.
cpuN := runtime.GOMAXPROCS(0)
var wg sync.WaitGroup
for i := 0; i < cpuN; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for j := 0; j < b.N/cpuN; j++ {
if err := s.SetAttrs(uint64(j%n), map[string]interface{}{"A": int64(100), "B": "foo", "D": 100.2}); err != nil {
b.Fatal(err)
}
}
}()
}
wg.Wait()
}
// MustOpenAttrStore returns a new, opened attribute store at a temporary path. Panic on error.
func MustOpenAttrStore() *AttrStore {
s := NewAttrStore()
if err := s.Open(); err != nil {
panic(err)
}
return s
}
// Close closes the database and removes the underlying data.
func (s *AttrStore) Close() error {
defer os.RemoveAll(s.Path())
return s.AttrStore.Close()
}

19
test/client.go Normal file
View file

@ -0,0 +1,19 @@
package test
import (
"github.com/pilosa/pilosa"
)
// Client represents a test wrapper for pilosa.Client.
type Client struct {
*pilosa.Client
}
// MustNewClient returns a new instance of Client. Panic on error.
func MustNewClient(host string) *Client {
c, err := pilosa.NewClient(host)
if err != nil {
panic(err)
}
return &Client{Client: c}
}

40
test/cluster.go Normal file
View file

@ -0,0 +1,40 @@
package test
import (
"fmt"
"github.com/pilosa/pilosa"
)
// NewCluster returns a cluster with n nodes and uses a mod-based hasher.
func NewCluster(n int) *pilosa.Cluster {
c := pilosa.NewCluster()
c.ReplicaN = 1
c.Hasher = NewModHasher()
for i := 0; i < n; i++ {
c.Nodes = append(c.Nodes, &pilosa.Node{
Host: fmt.Sprintf("host%d", i),
})
}
return c
}
// ModHasher represents a simple, mod-based hashing.
type ModHasher struct{}
// NewModHasher returns a new instance of ModHasher with n buckets.
func NewModHasher() *ModHasher { return &ModHasher{} }
func (*ModHasher) Hash(key uint64, n int) int { return int(key) % n }
// ConstHasher represents hash that always returns the same index.
type ConstHasher struct {
i int
}
// NewConstHasher returns a new instance of ConstHasher that always returns i.
func NewConstHasher(i int) *ConstHasher { return &ConstHasher{i: i} }
func (h *ConstHasher) Hash(key uint64, n int) int { return h.i }

32
test/executor.go Normal file
View file

@ -0,0 +1,32 @@
package test
import (
"strings"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/pql"
)
// Executor represents a test wrapper for pilosa.Executor.
type Executor struct {
*pilosa.Executor
}
// NewExecutor returns a new instance of Executor.
// The executor always matches the hostname of the first cluster node.
func NewExecutor(holder *pilosa.Holder, cluster *pilosa.Cluster) *Executor {
e := &Executor{Executor: pilosa.NewExecutor()}
e.Holder = holder
e.Cluster = cluster
e.Host = cluster.Nodes[0].Host
return e
}
// MustParse parses s into a PQL query. Panic on error.
func MustParse(s string) *pql.Query {
q, err := pql.NewParser(strings.NewReader(s)).Parse()
if err != nil {
panic(err)
}
return q
}

127
test/fragment.go Normal file
View file

@ -0,0 +1,127 @@
package test
import (
"io/ioutil"
"os"
"github.com/pilosa/pilosa"
)
// SliceWidth is a helper reference to use when testing.
const SliceWidth = pilosa.SliceWidth
// Fragment is a test wrapper for pilosa.Fragment.
type Fragment struct {
*pilosa.Fragment
RowAttrStore *AttrStore
}
// NewFragment returns a new instance of Fragment with a temporary path.
func NewFragment(index, frame, view string, slice uint64, cacheType string) *Fragment {
file, err := ioutil.TempFile("", "pilosa-fragment-")
if err != nil {
panic(err)
}
file.Close()
f := &Fragment{
Fragment: pilosa.NewFragment(file.Name(), index, frame, view, slice),
RowAttrStore: MustOpenAttrStore(),
}
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
return f
}
// MustOpenFragment creates and opens an fragment at a temporary path. Panic on error.
func MustOpenFragment(index, frame, view string, slice uint64, cacheType string) *Fragment {
if cacheType == "" {
cacheType = pilosa.DefaultCacheType
}
f := NewFragment(index, frame, view, slice, cacheType)
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the fragment and removes all underlying data.
func (f *Fragment) Close() error {
defer os.Remove(f.Path())
defer os.Remove(f.CachePath())
defer f.RowAttrStore.Close()
return f.Fragment.Close()
}
// Reopen closes the fragment and reopens it as a new instance.
func (f *Fragment) Reopen() error {
cacheType := f.Fragment.CacheType
path := f.Path()
if err := f.Fragment.Close(); err != nil {
return err
}
f.Fragment = pilosa.NewFragment(path, f.Index(), f.Frame(), f.View(), f.Slice())
f.Fragment.CacheType = cacheType
f.Fragment.RowAttrStore = f.RowAttrStore.AttrStore
if err := f.Open(); err != nil {
return err
}
return nil
}
// MustSetBits sets bits on a row. Panic on error.
// This function does not accept a timestamp or quantum.
func (f *Fragment) MustSetBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.SetBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// MustClearBits clears bits on a row. Panic on error.
func (f *Fragment) MustClearBits(rowID uint64, columnIDs ...uint64) {
for _, columnID := range columnIDs {
if _, err := f.ClearBit(rowID, columnID); err != nil {
panic(err)
}
}
}
// RowAttrStore provides simple storage for attributes.
type RowAttrStore struct {
attrs map[uint64]map[string]interface{}
}
// NewRowAttrStore returns a new instance of RowAttrStore.
func NewRowAttrStore() *RowAttrStore {
return &RowAttrStore{
attrs: make(map[uint64]map[string]interface{}),
}
}
// RowAttrs returns the attributes set to a row id.
func (s *RowAttrStore) RowAttrs(id uint64) (map[string]interface{}, error) {
return s.attrs[id], nil
}
// SetRowAttrs assigns a set of attributes to a row id.
func (s *RowAttrStore) SetRowAttrs(id uint64, m map[string]interface{}) {
s.attrs[id] = m
}
// GenerateImportFill generates a set of bits pairs that evenly fill a fragment chunk.
func GenerateImportFill(rowN int, pct float64) (rowIDs, columnIDs []uint64) {
ipct := int(pct * 100)
for i := 0; i < SliceWidth*rowN; i++ {
if i%100 >= ipct {
continue
}
rowIDs = append(rowIDs, uint64(i%SliceWidth))
columnIDs = append(columnIDs, uint64(i/SliceWidth))
}
return
}

92
test/frame.go Normal file
View file

@ -0,0 +1,92 @@
package test
import (
"io/ioutil"
"os"
"testing"
"time"
"github.com/pilosa/pilosa"
)
// Frame represents a test wrapper for pilosa.Frame.
type Frame struct {
*pilosa.Frame
}
// NewFrame returns a new instance of Frame d/0.
func NewFrame() *Frame {
path, err := ioutil.TempDir("", "pilosa-frame-")
if err != nil {
panic(err)
}
frame, err := pilosa.NewFrame(path, "i", "f")
if err != nil {
panic(err)
}
return &Frame{Frame: frame}
}
// MustOpenFrame returns a new, opened frame at a temporary path. Panic on error.
func MustOpenFrame() *Frame {
f := NewFrame()
if err := f.Open(); err != nil {
panic(err)
}
return f
}
// Close closes the frame and removes the underlying data.
func (f *Frame) Close() error {
defer os.RemoveAll(f.Path())
return f.Frame.Close()
}
// Reopen closes the index and reopens it.
func (f *Frame) Reopen() error {
var err error
if err := f.Frame.Close(); err != nil {
return err
}
path, index, name := f.Path(), f.Index(), f.Name()
f.Frame, err = pilosa.NewFrame(path, index, name)
if err != nil {
return err
}
if err := f.Open(); err != nil {
return err
}
return nil
}
// MustSetBit sets a bit on the frame. Panic on error.
func (f *Frame) MustSetBit(view string, rowID, columnID uint64, t *time.Time) (changed bool) {
changed, err := f.SetBit(view, rowID, columnID, t)
if err != nil {
panic(err)
}
return changed
}
// Ensure frame can set its cache
func TestFrame_SetCacheSize(t *testing.T) {
f := MustOpenFrame()
defer f.Close()
cacheSize := uint32(100)
// Set & retrieve frame cache size.
if err := f.SetCacheSize(cacheSize); err != nil {
t.Fatal(err)
} else if q := f.CacheSize(); q != cacheSize {
t.Fatalf("unexpected frame cache size: %d", q)
}
// Reload frame and verify that it is persisted.
if err := f.Reopen(); err != nil {
t.Fatal(err)
} else if q := f.CacheSize(); q != cacheSize {
t.Fatalf("unexpected frame cache size (reopen): %d", q)
}
}

144
test/handler.go Normal file
View file

@ -0,0 +1,144 @@
package test
import (
"context"
"encoding/json"
"errors"
"io"
"io/ioutil"
"net/http"
"net/http/httptest"
"net/url"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
)
// Handler represents a test wrapper for pilosa.Handler.
type Handler struct {
*pilosa.Handler
Executor HandlerExecutor
}
// NewHandler returns a new instance of Handler.
func NewHandler() *Handler {
h := &Handler{
Handler: pilosa.NewHandler(),
}
h.Handler.Executor = &h.Executor
h.Handler.LogOutput = ioutil.Discard
// Handler test messages can no-op.
h.Broadcaster = pilosa.NopBroadcaster
return h
}
// HandlerExecutor is a mock implementing pilosa.Handler.Executor.
type HandlerExecutor struct {
cluster *pilosa.Cluster
ExecuteFn func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error)
}
func (c *HandlerExecutor) Cluster() *pilosa.Cluster { return c.cluster }
func (c *HandlerExecutor) Execute(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return c.ExecuteFn(ctx, index, query, slices, opt)
}
// Server represents a test wrapper for httptest.Server.
type Server struct {
*httptest.Server
Handler *Handler
}
// NewServer returns a test server running on a random port.
func NewServer() *Server {
s := &Server{
Handler: NewHandler(),
}
s.Server = httptest.NewServer(s.Handler.Handler)
// Update handler to use hostname.
s.Handler.Host = s.Host()
// Handler test messages can no-op.
s.Handler.Broadcaster = pilosa.NopBroadcaster
// Create a default cluster on the handler
s.Handler.Cluster = NewCluster(1)
s.Handler.Cluster.Nodes[0].Host = s.Host()
return s
}
// LocalStatus returns the state of the local node as well as the
// holder (indexes/frames) according to the local node.
func (s *Server) LocalStatus() (proto.Message, error) {
if s.Handler.Holder == nil {
return nil, errors.New("Server.Holder is nil")
}
ns := internal.NodeStatus{
Host: s.Handler.Handler.Host,
State: pilosa.NodeStateUp,
Indexes: pilosa.EncodeIndexes(s.Handler.Holder.Indexes()),
}
// Append Slice list per this Node's indexes
for _, index := range ns.Indexes {
index.Slices = s.Handler.Cluster.OwnsSlices(index.Name, index.MaxSlice, s.Handler.Host)
}
return &ns, nil
}
// ClusterStatus returns the NodeState for all nodes in the cluster.
func (s *Server) ClusterStatus() (proto.Message, error) {
// Assuming we are only testing this with one Node
// So just return its status
return s.LocalStatus()
}
// HandleRemoteStatus just need to implement a nop to complete the Interface
func (s *Server) HandleRemoteStatus(pb proto.Message) error { return nil }
// Host returns the hostname of the running server.
func (s *Server) Host() string { return MustParseURLHost(s.URL) }
// MustParseURLHost parses rawurl and returns the hostname. Panic on error.
func MustParseURLHost(rawurl string) string {
u, err := url.Parse(rawurl)
if err != nil {
panic(err)
}
return u.Host
}
// MustNewHTTPRequest creates a new HTTP request. Panic on error.
func MustNewHTTPRequest(method, urlStr string, body io.Reader) *http.Request {
req, err := http.NewRequest(method, urlStr, body)
if err != nil {
panic(err)
}
return req
}
// MustMarshalJSON marshals v to JSON. Panic on error.
func MustMarshalJSON(v interface{}) []byte {
buf, err := json.Marshal(v)
if err != nil {
panic(err)
}
return buf
}
// MustReadAll reads a reader into a buffer and returns it. Panic on error.
func MustReadAll(r io.Reader) []byte {
buf, err := ioutil.ReadAll(r)
if err != nil {
panic(err)
}
return buf
}

115
test/holder.go Normal file
View file

@ -0,0 +1,115 @@
package test
import (
"bytes"
"io/ioutil"
"os"
"github.com/pilosa/pilosa"
)
// Holder is a test wrapper for pilosa.Holder.
type Holder struct {
*pilosa.Holder
LogOutput bytes.Buffer
}
// NewHolder returns a new instance of Holder with a temporary path.
func NewHolder() *Holder {
path, err := ioutil.TempDir("", "pilosa-")
if err != nil {
panic(err)
}
h := &Holder{Holder: pilosa.NewHolder()}
h.Path = path
h.Holder.LogOutput = &h.LogOutput
return h
}
// MustOpenHolder creates and opens a holder at a temporary path. Panic on error.
func MustOpenHolder() *Holder {
h := NewHolder()
if err := h.Open(); err != nil {
panic(err)
}
return h
}
// Close closes the holder and removes all underlying data.
func (h *Holder) Close() error {
defer os.RemoveAll(h.Path)
return h.Holder.Close()
}
// Reopen closes the holder and instantiates and opens a new holder.
func (h *Holder) Reopen() error {
if err := h.Holder.Close(); err != nil {
return err
}
path, logOutput := h.Path, h.Holder.LogOutput
h.Holder = pilosa.NewHolder()
h.Holder.Path = path
h.Holder.LogOutput = logOutput
if err := h.Holder.Open(); err != nil {
return err
}
return nil
}
// MustCreateIndexIfNotExists returns a given index. Panic on error.
func (h *Holder) MustCreateIndexIfNotExists(index string, opt pilosa.IndexOptions) *Index {
idx, err := h.Holder.CreateIndexIfNotExists(index, opt)
if err != nil {
panic(err)
}
return &Index{Index: idx}
}
// MustCreateFrameIfNotExists returns a given frame. Panic on error.
func (h *Holder) MustCreateFrameIfNotExists(index, frame string) *Frame {
f, err := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{}).CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
if err != nil {
panic(err)
}
return f
}
// MustCreateFragmentIfNotExists returns a given fragment. Panic on error.
func (h *Holder) MustCreateFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{})
if err != nil {
panic(err)
}
v, err := f.CreateViewIfNotExists(view)
if err != nil {
panic(err)
}
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
panic(err)
}
return &Fragment{Fragment: frag}
}
// MustCreateRankedFragmentIfNotExists returns a given fragment with a ranked cache. Panic on error.
func (h *Holder) MustCreateRankedFragmentIfNotExists(index, frame, view string, slice uint64) *Fragment {
idx := h.MustCreateIndexIfNotExists(index, pilosa.IndexOptions{})
f, err := idx.CreateFrameIfNotExists(frame, pilosa.FrameOptions{CacheType: pilosa.CacheTypeRanked})
if err != nil {
panic(err)
}
v, err := f.CreateViewIfNotExists(view)
if err != nil {
panic(err)
}
frag, err := v.CreateFragmentIfNotExists(slice)
if err != nil {
panic(err)
}
return &Fragment{Fragment: frag}
}

91
test/index.go Normal file
View file

@ -0,0 +1,91 @@
package test
import (
"io/ioutil"
"os"
"testing"
"github.com/pilosa/pilosa"
)
// Index represents a test wrapper for pilosa.Index.
type Index struct {
*pilosa.Index
}
// NewIndex returns a new instance of Index.
func NewIndex() *Index {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := pilosa.NewIndex(path, "i")
if err != nil {
panic(err)
}
return &Index{Index: index}
}
// MustOpenIndex returns a new, opened index at a temporary path. Panic on error.
func MustOpenIndex() *Index {
index := NewIndex()
if err := index.Open(); err != nil {
panic(err)
}
return index
}
// Close closes the index and removes the underlying data.
func (i *Index) Close() error {
defer os.RemoveAll(i.Path())
return i.Index.Close()
}
// Reopen closes the index and reopens it.
func (i *Index) Reopen() error {
var err error
if err := i.Index.Close(); err != nil {
return err
}
path, name := i.Path(), i.Name()
i.Index, err = pilosa.NewIndex(path, name)
if err != nil {
return err
}
if err := i.Open(); err != nil {
return err
}
return nil
}
// CreateFrame creates a frame with the given options.
func (i *Index) CreateFrame(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrame(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
}
// CreateFrameIfNotExists creates a frame with the given options if it doesn't exist.
func (i *Index) CreateFrameIfNotExists(name string, opt pilosa.FrameOptions) (*Frame, error) {
f, err := i.Index.CreateFrameIfNotExists(name, opt)
if err != nil {
return nil, err
}
return &Frame{Frame: f}, nil
}
// Ensure index can delete a frame.
func TestIndex_InvalidName(t *testing.T) {
path, err := ioutil.TempDir("", "pilosa-index-")
if err != nil {
panic(err)
}
index, err := pilosa.NewIndex(path, "ABC")
if index != nil {
t.Fatalf("unexpected index name %s", index)
}
}

5
view_test.go
View file

@ -19,12 +19,13 @@ import (
"os"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/test"
)
// View is a test wrapper for pilosa.View.
type View struct {
*pilosa.View
RowAttrStore *AttrStore
RowAttrStore *test.AttrStore
}
// NewView returns a new instance of View with a temporary path.
@ -37,7 +38,7 @@ func NewView(index, frame, name string) *View {
v := &View{
View: pilosa.NewView(file.Name(), index, frame, name, pilosa.DefaultCacheSize),
RowAttrStore: MustOpenAttrStore(),
RowAttrStore: test.MustOpenAttrStore(),
}
v.View.RowAttrStore = v.RowAttrStore.AttrStore
return v