scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-07 17:15:56 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'develop' into check-header

Browse source
This commit is contained in:
Travis Turner 2018-06-25 11:41:07 -05:00 • committed by GitHub
commit 2e035ee837
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 899 additions and 1263 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
api.go
View file

@ -46,16 +46,43 @@ type API struct {
Cluster *Cluster
TranslateStore TranslateStore
Logger Logger
server *Server
}
// APIOption is a functional option type for pilosa.API
type APIOption func(*API) error
func OptAPIServer(s *Server) APIOption {
return func(a *API) error {
a.server = s
a.Executor = s.executor
a.TranslateStore = s.translateFile
a.Holder = s.holder
a.Broadcaster = s
a.BroadcastHandler = s
a.StatusHandler = s
a.Cluster = s.Cluster
a.Logger = s.logger
return nil
}
}
// NewAPI returns a new API instance.
func NewAPI() *API {
return &API{
func NewAPI(opts ...APIOption) (*API, error) {
api := &API{
Broadcaster: NopBroadcaster,
//BroadcastHandler: NopBroadcastHandler, // TODO: implement the nop
//StatusHandler: NopStatusHandler, // TODO: implement the nop
Logger: NopLogger,
}
for _, opt := range opts {
err := opt(api)
if err != nil {
return nil, errors.Wrap(err, "applying option")
}
}
return api, nil
}
// validAPIMethods specifies the api methods that are valid for each

2
cluster.go
View file

@ -914,7 +914,7 @@ func (c *Cluster) open() error {
return fmt.Errorf("sending restart NodeJoin: %v", err)
}
c.Logger.Printf("wait for joining to complete")
c.Logger.Printf("%v wait for joining to complete", c.Node.ID)
<-c.joining
c.Logger.Printf("joining has completed")
}

2
cmd/server_test.go
View file

@ -15,7 +15,6 @@
package cmd_test
import (
"errors"
"io/ioutil"
"strings"
"testing"
@ -24,6 +23,7 @@ import (
"github.com/pilosa/pilosa/cmd"
_ "github.com/pilosa/pilosa/test"
"github.com/pilosa/pilosa/toml"
"github.com/pkg/errors"
)
func TestServerHelp(t *testing.T) {

18
ctl/export_test.go
View file

@ -44,22 +44,16 @@ func TestExportCommand_Validation(t *testing.T) {
}
func TestExportCommand_Run(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewExportCommand(stdin, stdout, stderr)
hostport := cmd.Server.URI.HostPort()
cm.Host = hostport
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", "http://"+hostport+"/index/i/field/f", strings.NewReader("")))
cm.Index = "i"
cm.Field = "f"

55
ctl/import_test.go
View file

@ -51,7 +51,6 @@ func TestImportCommand_Validation(t *testing.T) {
}
func TestImportCommand_Run(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
@ -62,15 +61,8 @@ func TestImportCommand_Run(t *testing.T) {
t.Fatal(err)
}
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
cmd := test.MustRunMainWithCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
cm.Index = "i"
cm.Field = "f"
@ -84,7 +76,6 @@ func TestImportCommand_Run(t *testing.T) {
// Ensure that the ImportValue path runs.
func TestImportCommand_RunValue(t *testing.T) {
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
@ -95,18 +86,11 @@ func TestImportCommand_RunValue(t *testing.T) {
t.Fatal(err)
}
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
cmd := test.MustRunMainWithCluster(t, 1)[0]
cm.Host = cmd.Server.URI.HostPort()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max": 100}}`)))
cm.Index = "i"
cm.Field = "f"
@ -118,20 +102,12 @@ func TestImportCommand_RunValue(t *testing.T) {
}
func TestImportCommand_InvalidFile(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cmd := test.MustRunMainWithCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
cm := NewImportCommand(stdin, stdout, stderr)
cm.Host = s.Host()
cm.Host = cmd.Server.URI.HostPort()
cm.Index = "i"
cm.Field = "f"
file, err := ioutil.TempFile("", "import.csv")
@ -200,6 +176,7 @@ func GetIO(buf bytes.Buffer) (io.Reader, io.Writer, io.Writer) {
}
func TestImportCommand_BugOverwriteValue(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
buf := bytes.Buffer{}
stdin, stdout, stderr := GetIO(buf)
@ -211,18 +188,10 @@ func TestImportCommand_BugOverwriteValue(t *testing.T) {
t.Fatal(err)
}
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
defer s.Close()
cm.Host = cmd.Server.Addr().String()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
cm.Host = s.Host()
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`)))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i", strings.NewReader("")))
http.DefaultClient.Do(MustNewHTTPRequest("POST", "http://"+cm.Host+"/index/i/field/f", strings.NewReader(`{"options":{"type": "int", "min": 0, "max":2147483648 }}`)))
cm.Index = "i"
cm.Field = "f"

6
executor.go
View file

@ -1618,6 +1618,9 @@ func (e *Executor) translateCall(index string, idx *Index, c *pql.Call) error {
// Translate row key, if field is specified & key exists.
if fieldName != "" {
field := idx.Field(fieldName)
if field == nil {
return ErrFieldNotFound
}
if field.Keys() {
if value := callArgString(c, rowKey); value != "" {
ids, err := e.TranslateStore.TranslateRowsToUint64(index, fieldName, []string{value})
@ -1659,6 +1662,9 @@ func (e *Executor) translateResult(index string, idx *Index, call *pql.Call, res
case []Pair:
if fieldName := callArgString(call, "_field"); fieldName != "" {
field := idx.Field(fieldName)
if field == nil {
return nil, ErrFieldNotFound
}
if field.Keys() {
other := make([]Pair, len(result))
for i := range result {

11
executor_test.go
View file

@ -1042,6 +1042,8 @@ func TestExecutor_Execute_BSIGroupRange(t *testing.T) {
// Ensure a remote query can return a row.
func TestExecutor_Execute_Remote_Row(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
@ -1090,6 +1092,8 @@ func TestExecutor_Execute_Remote_Row(t *testing.T) {
// Ensure a remote query can return a count.
func TestExecutor_Execute_Remote_Count(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
@ -1125,6 +1129,8 @@ func TestExecutor_Execute_Remote_Count(t *testing.T) {
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
c.ReplicaN = 2
@ -1178,6 +1184,8 @@ func TestExecutor_Execute_Remote_SetBit(t *testing.T) {
// Ensure a remote query can set columns on multiple nodes.
func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
c.ReplicaN = 2
@ -1233,6 +1241,8 @@ func TestExecutor_Execute_Remote_SetBit_With_Timestamp(t *testing.T) {
// Ensure a remote query can return a top-n query.
func TestExecutor_Execute_Remote_TopN(t *testing.T) {
t.Skip() // Until test.NewServer() works
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.
@ -1300,6 +1310,7 @@ func TestExecutor_Execute_Remote_TopN(t *testing.T) {
// Ensure a remote query can set RowAttrs
func TestExecutor_Execute_Remote_SetRowAttrs(t *testing.T) {
t.Skip("test.NewServer broken")
c := pilosa.NewTestCluster(2)
// Create secondary server and update second cluster node.

33
fragment_internal_test.go
View file

@ -213,39 +213,6 @@ func TestFragment_SetValue(t *testing.T) {
t.Fatal(err)
}
})
t.Run("Crash", func(t *testing.T) {
f := mustOpenFragment("i", "f", ViewStandard, 0, "")
defer f.Close()
// Set value.
if changed, err := f.setValue(0, 32, 17); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
if changed, err := f.setValue(0, 32, 16); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
if changed, err := f.setValue(0, 32, 19); err != nil {
t.Fatal(err)
} else if !changed {
t.Fatal("expected change")
}
// Read value.
if value, exists, err := f.value(0, 32); err != nil {
t.Fatal(err)
} else if value != 19 {
t.Fatalf("unexpected value: %d", value)
} else if !exists {
t.Fatal("expected to exist")
}
})
}
// Ensure a fragment can sum values.

17
handler.go
View file

@ -2,7 +2,6 @@ package pilosa
import (
"encoding/json"
"net"
)
// QueryRequest represent a request to process a query.
@ -61,18 +60,18 @@ func (resp *QueryResponse) MarshalJSON() ([]byte, error) {
}
type Handler interface {
Serve(ln net.Listener, closing <-chan struct{})
GetAPI() *API
Serve() error
Close() error
}
type NopHandler struct{}
type nopHandler struct{}
func (n *NopHandler) Serve(ln net.Listener, closing <-chan struct{}) {}
func (n *NopHandler) GetAPI() *API {
func (n nopHandler) Serve() error {
return nil
}
func NewNopHandler() Handler {
return &NopHandler{}
func (n nopHandler) Close() error {
return nil
}
var NopHandler Handler = nopHandler{}

2
holder_test.go
View file

@ -350,6 +350,8 @@ func TestHolder_DeleteIndex(t *testing.T) {
// Ensure holder can sync with a remote holder.
func TestHolderSyncer_SyncHolder(t *testing.T) {
t.Skip() // Until test.NewServer() works
s := test.NewServer()
defer s.Close()

47
http/client_test.go
View file

@ -52,6 +52,8 @@ func init() {
// Test distributed TopN Row count across 3 nodes.
func TestClient_MultiNode(t *testing.T) {
t.Skip() // Until test.NewServer() works
cluster := test.NewCluster(3)
s, hldr := createCluster(cluster)
@ -217,21 +219,17 @@ func TestClient_MultiNode(t *testing.T) {
// Ensure client can bulk import data.
func TestClient_Import(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
cmd := test.MustRunMainWithCluster(t, 1)[0]
host := cmd.Server.Addr().String()
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
// Load bitmap into cache to ensure cache gets updated.
hldr.SetBit("i", "f", 1, 0) // set a bit so the view gets created.
hldr.Row("i", "f", 0)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := MustNewClient(s.Host(), defaultClient)
c := MustNewClient(host, defaultClient)
if err := c.Import(context.Background(), "i", "f", 0, []pilosa.Bit{
{RowID: 0, ColumnID: 1},
{RowID: 0, ColumnID: 5},
@ -251,11 +249,12 @@ func TestClient_Import(t *testing.T) {
// Ensure client can bulk import value data.
func TestClient_ImportValue(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
cmd := test.MustRunMainWithCluster(t, 1)[0]
host := cmd.Server.Addr().String()
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
fldName := "f"
fo := pilosa.FieldOptions{
Type: pilosa.FieldTypeInt,
Min: -100,
@ -269,14 +268,8 @@ func TestClient_ImportValue(t *testing.T) {
t.Fatal(err)
}
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Send import request.
c := MustNewClient(s.Host(), defaultClient)
c := MustNewClient(host, defaultClient)
if err := c.ImportValue(context.Background(), "i", "f", 0, []pilosa.FieldValue{
{ColumnID: 1, Value: -10},
{ColumnID: 2, Value: 20},
@ -328,24 +321,16 @@ func TestClient_ImportValue(t *testing.T) {
// Ensure client can retrieve a list of all checksums for blocks in a fragment.
func TestClient_FragmentBlocks(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
cmd := test.MustRunMainWithCluster(t, 1)[0]
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
// Set two bits on blocks 0 & 3.
hldr.SetBit("i", "f", 0, 1)
hldr.SetBit("i", "f", pilosa.HashBlockSize*3, 100)
// Set a bit on a different slice.
hldr.SetBit("i", "f", 0, 1)
s := test.NewServer()
defer s.Close()
s.Handler.API.Cluster = test.NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
s.Handler.API.Holder = hldr.Holder
// Retrieve blocks.
c := MustNewClient(s.Host(), defaultClient)
c := MustNewClient(cmd.Server.Addr().String(), defaultClient)
blocks, err := c.FragmentBlocks(context.Background(), nil, "i", "f", 0)
if err != nil {
t.Fatal(err)

39
http/handler.go
View file

@ -15,6 +15,7 @@
package http
import (
"context"
"crypto/tls"
"encoding/json"
"expvar"
@ -53,6 +54,10 @@ type Handler struct {
API *pilosa.API
AllowedOrigins []string
ln net.Listener
server *http.Server
}
// externalPrefixFlag denotes endpoints that are intended to be exposed to clients.
@ -99,6 +104,13 @@ func OptHandlerLogger(logger pilosa.Logger) HandlerOption {
}
}
func OptHandlerListener(ln net.Listener) HandlerOption {
return func(h *Handler) error {
h.ln = ln
return nil
}
}
// NewHandler returns a new instance of Handler with a default logger.
func NewHandler(opts ...HandlerOption) (*Handler, error) {
handler := &Handler{
@ -114,19 +126,32 @@ func NewHandler(opts ...HandlerOption) (*Handler, error) {
}
}
if handler.API == nil {
return nil, errors.New("must pass OptHandlerAPI")
}
if handler.ln == nil {
return nil, errors.New("must pass OptHandlerListener")
}
handler.server = &http.Server{Handler: handler}
return handler, nil
}
func (h *Handler) Serve(ln net.Listener, closing <-chan struct{}) {
server := &http.Server{Handler: h}
go func() {
<-closing
server.Close()
}()
err := server.Serve(ln)
func (h *Handler) Serve() error {
err := h.server.Serve(h.ln)
if err != nil && err.Error() != "http: Server closed" {
h.Logger.Printf("HTTP handler terminated with error: %s\n", err)
return errors.Wrap(err, "serve http")
}
return nil
}
func (h *Handler) Close() error {
// TODO: timeout?
err := h.server.Shutdown(context.Background())
return errors.Wrap(err, "shutdown http server")
}
func (h *Handler) populateValidators() {

948
http/handler_test.go
View file

@ -15,952 +15,28 @@
package http_test
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/ioutil"
gohttp "net/http"
"net/http/httptest"
"reflect"
"strings"
"net"
"testing"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/pql"
"github.com/pilosa/pilosa/test"
)
func TestHandlerPanics(t *testing.T) {
h := test.MustNewHandler()
bufLogger := test.NewBufferLogger()
h.Handler.Logger = bufLogger
w := httptest.NewRecorder()
// will panic since Handler has no Holder set up
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/taxi", nil))
bufbytes, err := bufLogger.ReadAll()
if err != nil {
t.Fatalf("reading all logoutput: %v", err)
func TestHandlerOptions(t *testing.T) {
_, err := http.NewHandler()
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
if !bytes.Contains(bufbytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) {
t.Fatalf("expected panic in log, but got: %s", bufbytes)
_, err = http.NewHandler(http.OptHandlerAPI(&pilosa.API{}))
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
if w.Code != gohttp.StatusInternalServerError {
t.Fatalf("expected internal server error, but got: %v", w.Code)
}
bodyBytes := w.Body.Bytes()
if !bytes.Contains(bodyBytes, []byte("PANIC: runtime error: invalid memory address or nil pointer dereference")) {
t.Fatalf("response to client should have panic, but got %s", bodyBytes)
}
}
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_NotFound(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil))
if w.Code != gohttp.StatusNotFound {
t.Fatalf("invalid status: %d", w.Code)
}
}
// Ensure the handler can return the schema.
func TestHandler_Schema(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can return the status.
func TestHandler_Status(t *testing.T) {
s := test.NewServer()
hldr := test.MustOpenHolder()
defer s.Close()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.API.Cluster.SetState(pilosa.ClusterStateNormal)
h.API.StatusHandler = s
s.Handler = h
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"state":"NORMAL","nodes":[{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}],"localID":"node0"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
func TestHandler_Info(t *testing.T) {
s := test.NewServer()
defer s.Close()
h := test.MustNewHandler()
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can abort a cluster resize.
func TestHandler_ClusterResizeAbort(t *testing.T) {
t.Run("No resize job", func(t *testing.T) {
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Cluster.SetState(pilosa.ClusterStateResizing)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
if w.Code != gohttp.StatusOK {
bod, err := ioutil.ReadAll(w.Body)
t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err)
} else if body := w.Body.String(); body != `{"info":"complete current job: no resize job currently running"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
}
// Ensure the handler can return the maxslice map.
func TestHandler_MaxSlices(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1)
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2)
hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8)
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can accept URL arguments.
func TestHandler_Query_Args_URL(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Count(Row(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Row( id=100))")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can accept arguments via protobufs.
func TestHandler_Query_Args_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
if index != "idx0" {
t.Fatalf("unexpected index: %s", index)
} else if query.String() != `Count(Row(id=100))` {
t.Fatalf("unexpected query: %s", query.String())
} else if !reflect.DeepEqual(slices, []uint64{0, 1}) {
t.Fatalf("unexpected slices: %+v", slices)
}
return []interface{}{uint64(100)}, nil
}
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
Query: "Count(Row(id=100))",
Slices: []uint64{0, 1},
})
ln, err := net.Listen("tcp", ":0")
if err != nil {
t.Fatal(err)
}
// Generate protobuf request.
req := test.MustNewHTTPRequest("POST", "/index/idx0/query", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/x-protobuf")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Ensure the handler returns an error when parsing bad arguments.
func TestHandler_Query_Args_Err(t *testing.T) {
w := httptest.NewRecorder()
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=a,b", strings.NewReader("Row(id=100)")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
func TestHandler_Query_Params_Err(t *testing.T) {
w := httptest.NewRecorder()
test.MustNewHandler().ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1&db=sample", strings.NewReader("Row(id=100)")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query with a uint64 response as JSON.
func TestHandler_Query_Uint64_JSON(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("Count( Row( id=100))")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[100]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query with a uint64 response as protobufs.
func TestHandler_Query_Uint64_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{uint64(100)}, nil
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Count(Row(id=100))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if n := resp.Results[0].N; n != 100 {
t.Fatalf("unexpected n: %d", n)
}
}
// Ensure the handler can execute a query that returns a bitmap as JSON.
func TestHandler_Query_Bitmap_JSON(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1)
r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return []interface{}{r}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Row(id=100)")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can execute a query that returns a row with column attributes as JSON.
func TestHandler_Query_Row_ColumnAttrs_JSON(t *testing.T) {
hldr := test.NewHolder()
defer hldr.Close()
// Create index and set column attributes.
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(3, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(66, map[string]interface{}{"y": 123, "z": false}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
r := pilosa.NewRow(1, 3, 66, pilosa.SliceWidth+1)
r.Attrs = map[string]interface{}{"a": "b", "c": 1, "d": true}
return []interface{}{r}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query?columnAttrs=true", strings.NewReader("Row(id=100)")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1,3,66,1048577]}],"columnAttrs":[{"id":3,"attrs":{"x":"y"}},{"id":66,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can execute a query that returns a row as protobuf.
func TestHandler_Query_Row_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
r := pilosa.NewRow(1, pilosa.SliceWidth+1)
r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return []interface{}{r}, nil
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader("Row(id=100)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
}
}
// Ensure the handler can execute a query that returns a row with column attributes as protobuf.
func TestHandler_Query_Row_ColumnAttrs_Protobuf(t *testing.T) {
hldr := test.NewHolder()
defer hldr.Close()
// Create index and set column attributes.
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
r := pilosa.NewRow(1, pilosa.SliceWidth+1)
r.Attrs = map[string]interface{}{"a": "b", "c": int64(1), "d": true}
return []interface{}{r}, nil
}
// Encode request body.
buf, err := proto.Marshal(&internal.QueryRequest{
Query: "Row(id=100)",
ColumnAttrs: true,
})
if err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", bytes.NewReader(buf))
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{1, pilosa.SliceWidth + 1}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
}
if a := resp.ColumnAttrSets; len(a) != 1 {
t.Fatalf("unexpected column attributes length: %d", len(a))
} else if a[0].ID != 1 {
t.Fatalf("unexpected id: %d", a[0].ID)
} else if len(a[0].Attrs) != 1 {
t.Fatalf("unexpected column attr length: %d", len(a))
} else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
}
}
// Ensure the handler can execute a query that returns pairs as JSON.
func TestHandler_Query_Pairs_JSON(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
{ID: 3, Count: 4},
}}, nil
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`)))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[[{"id":1,"count":2},{"id":3,"count":4}]]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can execute a query that returns pairs as protobuf.
func TestHandler_Query_Pairs_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return []interface{}{[]pilosa.Pair{
{ID: 1, Count: 2},
{ID: 3, Count: 4},
}}, nil
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
}
// Ensure the handler can return an error as JSON.
func TestHandler_Query_Err_JSON(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`Row(id=100)`)))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"executing: marker"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
}
// Ensure the handler can return an error as protobuf.
func TestHandler_Query_Err_Protobuf(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
h.Executor.ExecuteFn = func(ctx context.Context, index string, query *pql.Query, slices []uint64, opt *pilosa.ExecOptions) ([]interface{}, error) {
return nil, errors.New("marker")
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i/query", strings.NewReader(`TopN(field=x, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if s := resp.Err; s != `executing: marker` {
t.Fatalf("unexpected error: %s", s)
}
}
// Ensure the handler returns "method not allowed" for non-POST queries.
func TestHandler_Query_MethodNotAllowed(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i/query", nil))
if w.Code != gohttp.StatusMethodNotAllowed {
t.Fatalf("invalid status: %d", w.Code)
}
}
// Ensure the handler returns an error if there is a parsing error..
func TestHandler_Query_ErrParse(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near IDENT (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" { // TODO not confident
t.Fatalf("unexpected body: \n%s", body)
}
}
// Ensure the handler can delete an index.
func TestHandler_Index_Delete(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Create index.
if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
// Send request to delete index.
resp, err := gohttp.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Verify body response.
if resp.StatusCode != gohttp.StatusOK {
t.Fatalf("unexpected status: %d", resp.StatusCode)
} else if buf, err := ioutil.ReadAll(resp.Body); err != nil {
t.Fatal(err)
} else if string(buf) != "{}\n" {
t.Fatalf("unexpected response body: %s", buf)
}
// Verify index is gone.
if hldr.Index("i") != nil {
t.Fatal("expected nil index")
}
}
// Ensure handler can delete a field.
func TestHandler_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
if _, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i0/field/f1", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i0").Field("f1"); f != nil {
t.Fatal("expected nil field")
}
}
// Ensure the handler can return data in differing blocks for an index.
func TestHandler_Index_AttrStore_Diff(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set attributes on the index.
index, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err != nil {
t.Fatal(err)
}
if err := index.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := index.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
// Retrieve block checksums.
blks, err := index.ColumnAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
// Remove block #0 and alter block 2's checksum.
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req, err := gohttp.NewRequest(
"POST",
s.URL+"/index/i/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
client := &gohttp.Client{}
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Read and validate body.
if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can return data in differing blocks for a field.
func TestHandler_Field_AttrStore_Diff(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// Set attributes on the index.
idx := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
f, err := idx.CreateFieldIfNotExists("meta", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
}
if err := f.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := f.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := f.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
// Retrieve block checksums.
blks, err := f.RowAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
// Remove block #0 and alter block 2's checksum.
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req, err := gohttp.NewRequest(
"POST",
s.URL+"/index/i/field/meta/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
client := &gohttp.Client{}
resp, err := client.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
// Read and validate body.
if body := string(test.MustReadAll(resp.Body)); body != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
}
// Ensure the handler can retrieve the version.
func TestHandler_Version(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/version", nil)
h.ServeHTTP(w, r)
version := pilosa.Version
if strings.HasPrefix(version, "v") {
version = version[1:]
}
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"version":"`+version+`"}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
}
// Ensure the handler can return a list of nodes for a fragment.
func TestHandler_Fragment_Nodes(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(3)
h.API.Cluster.ReplicaN = 2
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `[{"id":"node2","uri":{"scheme":"http","host":"host2"},"isCoordinator":false},{"id":"node0","uri":{"scheme":"http","host":"host0"},"isCoordinator":false}]`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
// invalid argument should return BadRequest
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
// index is required
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
// Ensure the handler can return expvars without panicking.
func TestHandler_Expvars(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Cluster = test.NewCluster(1)
h.API.Holder = hldr.Holder
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
func MustReadAll(r io.Reader) []byte {
buf, err := ioutil.ReadAll(r)
if err != nil {
panic(err)
}
return buf
}
func TestHandler_RecalculateCaches(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
h := test.MustNewHandler()
h.API.Holder = hldr.Holder
h.API.Cluster = test.NewCluster(1)
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil))
if w.Code != gohttp.StatusNoContent {
t.Fatalf("unexpected status code: %d", w.Code)
}
}
func TestHandler_CORS(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
// No CORS config present, so should fail
handler := test.MustNewHandler()
req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil)
req.Header.Add("Origin", "http://test/")
req.Header.Add("Access-Control-Request-Method", "POST")
w := httptest.NewRecorder()
handler.ServeHTTP(w, req)
result := w.Result()
// This handler does not support CORS, return Method Not Allowed (405)
if result.StatusCode != 405 {
t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode)
}
// CORS config should allow preflight response
handler = test.MustNewHandler(http.OptHandlerAllowedOrigins([]string{"http://test/"}))
w = httptest.NewRecorder()
handler.ServeHTTP(w, req)
result = w.Result()
if result.StatusCode != 200 {
t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode)
}
if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" {
t.Fatal("CORS header not present")
_, err = http.NewHandler(http.OptHandlerListener(ln))
if err == nil {
t.Fatalf("expected error making handler without options, got nil")
}
}

2
http/translator_test.go
View file

@ -15,6 +15,8 @@ import (
)
func TestTranslateStore_Reader(t *testing.T) {
t.Skip() // Until test.NewServer() works
// Ensure client can connect and stream the translate store data.
t.Run("OK", func(t *testing.T) {
t.Run("ServerDisconnect", func(t *testing.T) {

63
server.go
View file

@ -61,12 +61,10 @@ type Server struct {
clusterDisabled bool
// External
handler Handler
BroadcastReceiver BroadcastReceiver
systemInfo SystemInfo
gcNotifier GCNotifier
logger Logger
ln net.Listener
NodeID string
URI URI
@ -81,6 +79,11 @@ type Server struct {
dataDir string
}
// TODO: have this return an interface for Holder instead of concrete object?
func (s *Server) Holder() *Holder {
return s.holder
}
// ServerOption is a functional option type for pilosa.Server
type ServerOption func(s *Server) error
@ -126,13 +129,6 @@ func OptServerLongQueryTime(dur time.Duration) ServerOption {
}
}
func OptServerHandler(h Handler) ServerOption {
return func(s *Server) error {
s.handler = h
return nil
}
}
func OptServerMaxWritesPerRequest(n int) ServerOption {
return func(s *Server) error {
s.maxWritesPerRequest = n
@ -191,14 +187,6 @@ func OptServerDiagnosticsInterval(dur time.Duration) ServerOption {
}
}
func OptServerListener(ln net.Listener) ServerOption {
return func(s *Server) error {
s.ln = ln
return nil
}
}
func OptServerURI(uri *URI) ServerOption {
return func(s *Server) error {
s.URI = *uri
@ -264,11 +252,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
return nil, err
}
// update URI port with actual listener port. TODO this should probably be done outside of here.
if s.URI.Port() == 0 {
s.URI.SetPort(uint16(s.ln.Addr().(*net.TCPAddr).Port))
}
// Get or create NodeID.
s.NodeID = s.LoadNodeID()
// Set Cluster Node.
@ -293,8 +276,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
s.executor.Cluster = s.Cluster
s.executor.TranslateStore = s.translateFile
s.executor.MaxWritesPerRequest = s.maxWritesPerRequest
s.handler.GetAPI().Executor = s.executor
s.handler.GetAPI().TranslateStore = s.translateFile
return s, nil
}
@ -302,9 +283,6 @@ func NewServer(opts ...ServerOption) (*Server, error) {
// Open opens and initializes the server.
func (s *Server) Open() error {
s.logger.Printf("open server")
if s.ln == nil {
return errors.New("must pass a listener option to NewServer")
}
// Log startup
err := s.holder.logStartup()
@ -316,20 +294,9 @@ func (s *Server) Open() error {
s.Cluster.Broadcaster = s
s.Cluster.MaxWritesPerRequest = s.maxWritesPerRequest
// Initialize HTTP handler.
api := s.handler.GetAPI()
api.Holder = s.holder
api.Broadcaster = s
api.BroadcastHandler = s
api.StatusHandler = s
api.Cluster = s.Cluster
// Initialize Holder.
s.holder.Broadcaster = s
// Serve handler.
go s.handler.Serve(s.ln, s.closing)
// Start the BroadcastReceiver.
if err := s.BroadcastReceiver.Start(s); err != nil {
return fmt.Errorf("starting BroadcastReceiver: %v", err)
@ -370,9 +337,6 @@ func (s *Server) Close() error {
close(s.closing)
s.wg.Wait()
if s.ln != nil {
s.ln.Close()
}
if s.Cluster != nil {
s.Cluster.close()
}
@ -400,12 +364,21 @@ func (s *Server) LoadNodeID() string {
return nodeID
}
type pilosaAddr URI
func (p pilosaAddr) String() string {
uri := URI(p)
return uri.HostPort()
}
func (pilosaAddr) Network() string {
return "tcp"
}
// Addr returns the address of the listener.
func (s *Server) Addr() net.Addr {
if s.ln == nil {
return nil
}
return s.ln.Addr()
return pilosaAddr(s.URI)
}
func (s *Server) monitorAntiEntropy() {

599
server/handler_test.go Normal file
View file

@ -0,0 +1,599 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package server_test
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http/httptest"
"reflect"
"strings"
"testing"
gohttp "net/http"
"github.com/gogo/protobuf/proto"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/internal"
"github.com/pilosa/pilosa/test"
)
// Ensure the handler returns "not found" for invalid paths.
func TestHandler_Endpoints(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
h := cmd.Handler.(*http.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
t.Run("Not Found", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/no_such_path", nil))
if w.Code != gohttp.StatusNotFound {
t.Fatalf("invalid status: %d", w.Code)
}
})
t.Run("Info", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/info", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != fmt.Sprintf("{\"sliceWidth\":%d}\n", pilosa.SliceWidth) {
t.Fatalf("unexpected body: %s", body)
}
})
i0 := hldr.MustCreateIndexIfNotExists("i0", pilosa.IndexOptions{})
i1 := hldr.MustCreateIndexIfNotExists("i1", pilosa.IndexOptions{})
if f, err := i0.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if f, err := i1.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
} else if _, err := f.SetBit(0, 0, nil); err != nil {
t.Fatal(err)
}
if _, err := i0.CreateFieldIfNotExists("f0", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
t.Run("Schema", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/schema", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0"},{"name":"f1","views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","views":[{"name":"standard"}]}]}]}`+"\n" {
} else if body := w.Body.String(); body != `{"indexes":[{"name":"i0","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000}},{"name":"f1","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]},{"name":"i1","fields":[{"name":"f0","options":{"cacheType":"ranked","cacheSize":50000},"views":[{"name":"standard"}]}]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
t.Run("Status", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/status", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
ret := mustJSONDecode(t, w.Body)
if ret["state"].(string) != "NORMAL" {
t.Fatalf("wrong state from /status: %#v", ret)
}
if len(ret["nodes"].([]interface{})) != 1 {
t.Fatalf("wrong length nodes list: %#v", ret)
}
})
t.Run("Abort no resize job", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/cluster/resize/abort", nil))
if w.Code != gohttp.StatusInternalServerError {
bod, err := ioutil.ReadAll(w.Body)
t.Fatalf("unexpected status code: %d, bod: %s, readerr: %v", w.Code, bod, err)
}
// TODO need to test aborting a cluster resize job. this may not be the right place
})
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+1)
hldr.SetBit("i0", "f0", 30, (1*pilosa.SliceWidth)+2)
hldr.SetBit("i0", "f0", 30, (3*pilosa.SliceWidth)+4)
hldr.SetBit("i0", "f0", 31, 1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+1)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+2)
hldr.SetBit("i1", "f1", 40, (0*pilosa.SliceWidth)+8)
t.Run("Max Slice", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/slices/max", nil))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"standard":{"i0":3,"i1":0}}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
t.Run("Slices args", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1", strings.NewReader("Count(Row(f0=30))")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[2]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Slices args protobuf", func(t *testing.T) {
// Generate request body.
reqBody, err := proto.Marshal(&internal.QueryRequest{
Query: "Count(Row(f0=30))",
Slices: []uint64{0, 1},
})
if err != nil {
t.Fatal(err)
}
// Generate protobuf request.
req := test.MustNewHTTPRequest("POST", "/index/i0/query", bytes.NewReader(reqBody))
req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("Accept", "application/json")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[2]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Query args error", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=a,b", strings.NewReader("Count(Row(f0=30))")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"invalid slice argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Query params err", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?slices=0,1&db=sample", strings.NewReader("Count(Row(f0=30))")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"db is not a valid argument"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Uint64 protobuf", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Count(Row(f0=30))"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeUint64 {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if n := resp.Results[0].N; n != 3 {
t.Fatalf("unexpected n: %d", n)
}
})
t.Run("Row JSON", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[{"attrs":{},"columns":[1048577,1048578,3145732]}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
f0 := i0.Field("f0")
if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+1, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := i0.ColumnAttrStore().SetAttrs((1*pilosa.SliceWidth)+2, map[string]interface{}{"y": 123, "z": false}); err != nil {
t.Fatal(err)
} else if err := f0.RowAttrStore().SetAttrs(30, map[string]interface{}{"a": "b", "c": 1, "d": true}); err != nil {
t.Fatal(err)
}
t.Run("ColumnAttrs_JSON", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query?columnAttrs=true", strings.NewReader("Row(f0=30)")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d. body: %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{"results":[{"attrs":{"a":"b","c":1,"d":true},"columns":[1048577,1048578,3145732]}],"columnAttrs":[{"id":1048577,"attrs":{"x":"y"}},{"id":1048578,"attrs":{"y":123,"z":false}}]}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
t.Run("Row pbuf", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader("Row(f0=30)"))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
}
})
t.Run("Row columnattrs protobuf", func(t *testing.T) {
// Encode request body.
buf, err := proto.Marshal(&internal.QueryRequest{
Query: "Row(f0=30)",
ColumnAttrs: true,
})
if err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", bytes.NewReader(buf))
r.Header.Set("Content-Type", "application/x-protobuf")
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
if columns := resp.Results[0].Row.Columns; !reflect.DeepEqual(columns, []uint64{pilosa.SliceWidth + 1, pilosa.SliceWidth + 2, (3 * pilosa.SliceWidth) + 4}) {
t.Fatalf("unexpected columns: %+v", columns)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypeRow {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if attrs := resp.Results[0].Row.Attrs; len(attrs) != 3 {
t.Fatalf("unexpected attr length: %d", len(attrs))
} else if k, v := attrs[0].Key, attrs[0].StringValue; k != "a" || v != "b" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
} else if k, v := attrs[1].Key, attrs[1].IntValue; k != "c" || v != int64(1) {
t.Fatalf("unexpected attr[1]: %s=%v", k, v)
} else if k, v := attrs[2].Key, attrs[2].BoolValue; k != "d" || !v {
t.Fatalf("unexpected attr[2]: %s=%v", k, v)
}
if a := resp.ColumnAttrSets; len(a) != 2 {
t.Fatalf("unexpected column attributes length: %d", len(a))
} else if a[0].ID != pilosa.SliceWidth+1 {
t.Fatalf("unexpected id: %d", a[0].ID)
} else if len(a[0].Attrs) != 1 {
t.Fatalf("unexpected column attr length: %d", len(a))
} else if k, v := a[0].Attrs[0].Key, a[0].Attrs[0].StringValue; k != "x" || v != "y" {
t.Fatalf("unexpected attr[0]: %s=%v", k, v)
}
})
t.Run("Query Pairs JSON", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`)))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"results":[[{"id":30,"count":3},{"id":31,"count":1}]]}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Query Pairs protobuf", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`TopN(f0, n=2)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if rt := resp.Results[0].Type; rt != http.QueryResultTypePairs {
t.Fatalf("unexpected response type: %d", resp.Results[0].Type)
} else if a := resp.Results[0].GetPairs(); len(a) != 2 {
t.Fatalf("unexpected pair length: %d", len(a))
}
})
t.Run("Query err JSON", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Row(row=30)`)))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"executing: field not found"}`+"\n" {
t.Fatalf("unexpected body: %q", body)
}
})
t.Run("Query err protobuf", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("POST", "/index/i0/query", strings.NewReader(`Row(row=30)`))
r.Header.Set("Accept", "application/x-protobuf")
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
var resp internal.QueryResponse
if err := proto.Unmarshal(w.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
} else if s := resp.Err; s != `executing: field not found` {
t.Fatalf("unexpected error: %s", s)
}
})
t.Run("Method not allowed", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/index/i0/query", nil))
if w.Code != gohttp.StatusMethodNotAllowed {
t.Fatalf("invalid status: %d", w.Code)
}
})
t.Run("Err Parse", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/idx0/query?slices=0,1", strings.NewReader("bad_fn(")))
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
} else if body := w.Body.String(); body != `{"error":"parsing: parsing: \nparse error near IDENT (line 1 symbol 1 - line 1 symbol 4):\n\"bad\"\n"}`+"\n" {
t.Fatalf("unexpected body: %s", body)
}
})
t.Run("delete index", func(t *testing.T) {
hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
} else if w.Body.String() != "{}\n" {
t.Fatalf("unexpected response body: %s", w.Body.String())
}
// Verify index is gone.
if hldr.Index("i") != nil {
t.Fatal("expected nil index")
}
})
t.Run("Field delete", func(t *testing.T) {
i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := i.CreateFieldIfNotExists("f1", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f1", strings.NewReader("")))
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
} else if body := w.Body.String(); body != `{}`+"\n" {
t.Fatalf("unexpected body: %s", body)
} else if f := hldr.Index("i").Field("f1"); f != nil {
t.Fatal("expected nil field")
}
})
i := hldr.MustCreateIndexIfNotExists("i", pilosa.IndexOptions{})
if err := i.ColumnAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := i.ColumnAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := i.ColumnAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
t.Run("AttrStore Diff", func(t *testing.T) {
blks, err := i.ColumnAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req := test.MustNewHTTPRequest(
"POST",
"/index/i/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
}
// Read and validate body.
if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", w.Body.String())
}
})
meta, err := i.CreateFieldIfNotExists("meta", pilosa.FieldOptions{})
if err != nil {
t.Fatal(err)
}
if err := meta.RowAttrStore().SetAttrs(1, map[string]interface{}{"foo": 1, "bar": 2}); err != nil {
t.Fatal(err)
} else if err := meta.RowAttrStore().SetAttrs(100, map[string]interface{}{"x": "y"}); err != nil {
t.Fatal(err)
} else if err := meta.RowAttrStore().SetAttrs(200, map[string]interface{}{"snowman": "☃"}); err != nil {
t.Fatal(err)
}
t.Run("field attrstore diff", func(t *testing.T) {
blks, err := meta.RowAttrStore().Blocks()
if err != nil {
t.Fatal(err)
}
blks = blks[1:]
blks[1].Checksum = []byte("MISMATCHED_CHECKSUM")
// Send block checksums to determine diff.
req := test.MustNewHTTPRequest(
"POST",
"/index/i/field/meta/attr/diff",
strings.NewReader(`{"blocks":`+string(test.MustMarshalJSON(blks))+`}`),
)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "application/json")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d, body: %s", w.Code, w.Body.String())
}
// Read and validate body.
if w.Body.String() != `{"attrs":{"1":{"bar":2,"foo":1},"200":{"snowman":"☃"}}}`+"\n" {
t.Fatalf("unexpected body: %s", w.Body.String())
}
})
t.Run("Version", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/version", nil)
h.ServeHTTP(w, r)
version := strings.TrimPrefix(pilosa.Version, "v")
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
} else if w.Body.String() != `{"version":"`+version+`"}`+"\n" {
t.Fatalf("unexpected body: %q", w.Body.String())
}
})
t.Run("Fragment Nodes", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/fragment/nodes?index=i&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
body := mustJSONDecodeSlice(t, w.Body)
bmap := body[0].(map[string]interface{})
if bmap["isCoordinator"] != true {
t.Fatalf("expected true coordinator")
}
// invalid argument should return BadRequest
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?db=X&slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
// index is required
w = httptest.NewRecorder()
r = test.MustNewHTTPRequest("GET", "/fragment/nodes?slice=0", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusBadRequest {
t.Fatalf("unexpected status code: %d", w.Code)
}
})
t.Run("Expvars", func(t *testing.T) {
w := httptest.NewRecorder()
r := test.MustNewHTTPRequest("GET", "/debug/vars", nil)
h.ServeHTTP(w, r)
if w.Code != gohttp.StatusOK {
t.Fatalf("unexpected status code: %d", w.Code)
}
})
t.Run("Recalculate Caches", func(t *testing.T) {
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/recalculate-caches", nil))
if w.Code != gohttp.StatusNoContent {
t.Fatalf("unexpected status code: %d", w.Code)
}
})
t.Run("CORS", func(t *testing.T) {
req := test.MustNewHTTPRequest("OPTIONS", "/index/foo/query", nil)
req.Header.Add("Origin", "http://test/")
req.Header.Add("Access-Control-Request-Method", "POST")
w := httptest.NewRecorder()
h.ServeHTTP(w, req)
result := w.Result()
// This handler does not support CORS, return Method Not Allowed (405)
if result.StatusCode != 405 {
t.Fatalf("CORS preflight status should be 405, but is %v", result.StatusCode)
}
clus := test.MustRunMainWithCluster(t, 1, test.OptAllowedOrigins([]string{"http://test/"}))
w = httptest.NewRecorder()
h := clus[0].Handler.(*http.Handler).Handler
h.ServeHTTP(w, req)
result = w.Result()
if result.StatusCode != 200 {
t.Fatalf("CORS preflight status should be 200, but is %v", result.StatusCode)
}
if w.HeaderMap["Access-Control-Allow-Origin"][0] != "http://test/" {
t.Fatal("CORS header not present")
}
})
}
func mustJSONDecode(t *testing.T, r io.Reader) (ret map[string]interface{}) {
dec := json.NewDecoder(r)
err := dec.Decode(&ret)
if err != nil {
t.Fatalf("decoding response: %v", err)
}
return ret
}
func mustJSONDecodeSlice(t *testing.T, r io.Reader) (ret []interface{}) {
dec := json.NewDecoder(r)
err := dec.Decode(&ret)
if err != nil {
t.Fatalf("decoding response: %v", err)
}
return ret
}

56
server/server.go
View file

@ -73,6 +73,9 @@ type Command struct {
// Passed to the Gossip implementation.
logOutput io.Writer
logger loggerLogger
Handler pilosa.Handler
ln net.Listener
}
// NewCommand returns a new instance of Main.
@ -102,6 +105,12 @@ func (m *Command) Start() (err error) {
if err != nil {
return errors.Wrap(err, "setting up networking")
}
go func() {
err := m.Handler.Serve()
if err != nil {
m.logger.Printf("Handler serve error: %v", err)
}
}()
// Initialize server.
if err = m.Server.Open(); err != nil {
@ -164,18 +173,6 @@ func (m *Command) SetupServer() error {
}
m.logger.Printf("%s %s, build time %s\n", productName, pilosa.Version, pilosa.BuildTime)
api := pilosa.NewAPI()
api.Logger = m.logger
handler, err := http.NewHandler(
http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins),
http.OptHandlerAPI(api),
http.OptHandlerLogger(m.logger),
)
if err != nil {
return errors.Wrap(err, "wrapping handler")
}
uri, err := pilosa.AddressWithDefaults(m.Config.Bind)
if err != nil {
return errors.Wrap(err, "processing bind address")
@ -210,11 +207,16 @@ func (m *Command) SetupServer() error {
return errors.Wrap(err, "new stats client")
}
ln, err := getListener(*uri, TLSConfig)
m.ln, err = getListener(*uri, TLSConfig)
if err != nil {
return errors.Wrap(err, "getting listener")
}
// If port is 0, get auto-allocated port from listener
if uri.Port() == 0 {
uri.SetPort(uint16(m.ln.Addr().(*net.TCPAddr).Port))
}
c := http.GetHTTPClient(TLSConfig)
// Setup connection to primary store if this is a replica.
@ -234,18 +236,31 @@ func (m *Command) SetupServer() error {
pilosa.OptServerLogger(m.logger),
pilosa.OptServerAttrStoreFunc(boltdb.NewAttrStore),
pilosa.OptServerHandler(handler),
pilosa.OptServerSystemInfo(gopsutil.NewSystemInfo()),
pilosa.OptServerGCNotifier(gcnotify.NewActiveGCNotifier()),
pilosa.OptServerStatsClient(statsClient),
pilosa.OptServerListener(ln),
pilosa.OptServerURI(uri),
pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)),
pilosa.OptServerPrimaryTranslateStore(primaryTranslateStore),
pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts),
)
if err != nil {
return errors.Wrap(err, "new server")
}
api, err := pilosa.NewAPI(pilosa.OptAPIServer(m.Server))
if err != nil {
return errors.Wrap(err, "new api")
}
m.Handler, err = http.NewHandler(
http.OptHandlerAllowedOrigins(m.Config.Handler.AllowedOrigins),
http.OptHandlerAPI(api),
http.OptHandlerLogger(m.logger),
http.OptHandlerListener(m.ln),
)
return errors.Wrap(err, "new handler")
return errors.Wrap(err, "new server")
}
// SetupNetworking sets up internode communication based on the configuration.
@ -300,17 +315,16 @@ func (m *Command) SetupNetworking() error {
// Close shuts down the server.
func (m *Command) Close() error {
var logErr error
handlerErr := m.Handler.Close()
serveErr := m.Server.Close()
if closer, ok := m.logOutput.(io.Closer); ok {
logErr = closer.Close()
}
close(m.done)
if serveErr != nil && logErr != nil {
return fmt.Errorf("closing server: '%v', closing logs: '%v'", serveErr, logErr)
} else if logErr != nil {
return logErr
if serveErr != nil || logErr != nil || handlerErr != nil {
return fmt.Errorf("closing server: '%v', closing logs: '%v', closing handler: '%v'", serveErr, logErr, handlerErr)
}
return serveErr
return nil
}
// NewStatsClient creates a stats client from the config

4
server_test.go
View file

@ -27,7 +27,7 @@ import (
// pilosa.Server was not having its remoteClient field set by an option and so
// it was using a nil client in monitorAntiEntropy.
func TestMonitorAntiEntropy(t *testing.T) {
cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*1))
cluster := test.MustRunMainWithCluster(t, 3, test.OptAntiEntropyInterval(time.Millisecond*20))
client := cluster[1].Client()
err := client.CreateIndex(context.Background(), "balh", pilosa.IndexOptions{})
if err != nil {
@ -38,7 +38,7 @@ func TestMonitorAntiEntropy(t *testing.T) {
t.Fatalf("creating field: %v", err)
}
time.Sleep(time.Millisecond * 2)
time.Sleep(time.Millisecond * 40)
for _, m := range cluster {
err := m.Close()
if err != nil {

180
stats_test.go
View file

@ -16,12 +16,13 @@ package pilosa_test
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"time"
"github.com/pilosa/pilosa"
"github.com/pilosa/pilosa/http"
"github.com/pilosa/pilosa/test"
)
@ -207,116 +208,89 @@ func TestStatsCount_SetProfileAttrs(t *testing.T) {
}
}
func TestStatsCount_CreateIndex(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called := false
s.Handler.API.Holder.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "createIndex" {
t.Errorf("Expected createIndex, Results %s", name)
}
func TestStatsCount_APICalls(t *testing.T) {
cmd := test.MustRunMainWithCluster(t, 1)[0]
h := cmd.Handler.(*http.Handler).Handler
holder := cmd.Server.Holder()
hldr := test.Holder{Holder: holder}
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i", nil))
if !called {
t.Error("Count isn't called")
}
}
t.Run("create index", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "createIndex" {
t.Errorf("Expected createIndex, Results %s", name)
}
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
func TestStatsCount_DeleteIndex(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
t.Run("create field", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "createField" {
t.Errorf("Expected createField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("POST", "/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
// Create index.
if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
called := false
s.Handler.API.Holder.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "deleteIndex" {
t.Errorf("Expected deleteIndex, Results %s", name)
}
t.Run("delete field", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "deleteField" {
t.Errorf("Expected deleteField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
}
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
func TestStatsCount_CreateField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
t.Run("delete index", func(t *testing.T) {
called := false
hldr.Stats = &MockStats{
mockCount: func(name string, value int64, rate float64) {
if name != "deleteIndex" {
t.Errorf("Expected deleteIndex, Results %s", name)
}
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called = true
},
}
w := httptest.NewRecorder()
h.ServeHTTP(w, test.MustNewHTTPRequest("DELETE", "/index/i", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
})
// Create index.
if _, err := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{}); err != nil {
t.Fatal(err)
}
called := false
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "createField" {
t.Errorf("Expected createField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("POST", s.URL+"/index/i/field/f", nil))
if !called {
t.Error("Count isn't called")
}
}
func TestStatsCount_DeleteField(t *testing.T) {
hldr := test.MustOpenHolder()
defer hldr.Close()
s := test.NewServer()
s.Handler.API.Holder = hldr.Holder
defer s.Close()
called := false
// Create index.
indx, _ := hldr.CreateIndexIfNotExists("i", pilosa.IndexOptions{})
if _, err := indx.CreateFieldIfNotExists("test", pilosa.FieldOptions{}); err != nil {
t.Fatal(err)
}
s.Handler.API.Holder.Stats = &MockStats{
mockCountWithTags: func(name string, value int64, rate float64, index []string) {
if name != "deleteField" {
t.Errorf("Expected deleteField, Results %s", name)
}
if index[0] != "index:i" {
t.Errorf("Expected index:i, Results %s", index)
}
called = true
},
}
http.DefaultClient.Do(test.MustNewHTTPRequest("DELETE", s.URL+"/index/i/field/f", strings.NewReader("")))
if !called {
t.Error("Count isn't called")
}
}
type MockStats struct {

32
test/handler.go
View file

@ -45,9 +45,6 @@ func NewHandler(opts ...http.HandlerOption) (*Handler, error) {
h := &Handler{
Handler: handler,
}
h.API = pilosa.NewAPI()
h.Handler.API = h.API
h.Handler.API.Executor = &h.Executor
// Handler test messages can no-op.
h.API.Broadcaster = pilosa.NopBroadcaster
@ -84,22 +81,23 @@ type Server struct {
// NewServer returns a test server running on a random port.
func NewServer() *Server {
handler, err := NewHandler()
if err != nil {
panic(err)
}
s := &Server{
Handler: handler,
}
s.Server = httptest.NewServer(s.Handler.Handler)
return &Server{}
//handler, err := pilosa.NewHandler()
//if err != nil {
// panic(err)
//}
//s := &Server{
// Handler: handler,
//}
//s.Server = httptest.NewServer(s.Handler.Handler)
// Handler test messages can no-op.
s.Handler.API.Broadcaster = pilosa.NopBroadcaster
// Create a default cluster on the handler
s.Handler.API.Cluster = NewCluster(1)
s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
//// Handler test messages can no-op.
//s.Handler.API.Broadcaster = pilosa.NopBroadcaster
//// Create a default cluster on the handler
//s.Handler.API.Cluster = NewCluster(1)
//s.Handler.API.Cluster.Nodes[0].URI = s.HostURI()
return s
//return s
}
// LocalStatus exists so that test.Server implements StatusHandler.

15
test/pilosa.go
View file

@ -19,6 +19,7 @@ import (
"fmt"
"io"
"io/ioutil"
"log"
gohttp "net/http"
"os"
"strings"
@ -51,6 +52,13 @@ func OptAntiEntropyInterval(dur time.Duration) MainOpt {
}
}
func OptAllowedOrigins(origins []string) MainOpt {
return func(m *Main) error {
m.Config.Handler.AllowedOrigins = origins
return nil
}
}
// NewMain returns a new instance of Main with a temporary data directory and random port.
func NewMain(opts ...MainOpt) *Main {
path, err := ioutil.TempDir("", "pilosa-")
@ -221,6 +229,13 @@ func (m *Main) RunWithTransport(host string, bindPort int, joinSeeds []string) (
m.Server.Cluster.Static = false
go func() {
err := m.Handler.Serve()
if err != nil {
log.Printf("Handler serve error: %v", err)
}
}()
// Initialize server.
err = m.Server.Open()
if err != nil {