Enable authentication/authorization for featurebase tools

- Add auth-token for featurebase import, backup and restore - Add auth-token to http request - Create a cluster tests with auth enabled - Add test for import with auth enabled
2026-09-07 00:55:55 +00:00 · 2022-01-26 17:30:26 -06:00 · 2022-01-26 17:30:26 -06:00 · 0e1cf5bbbd
commit 0e1cf5bbbd
parent d7c3fa2a93
36 changed files with 1294 additions and 50 deletions
--- a/.gitlab/.gitlab-ci.yml
+++ b/.gitlab/.gitlab-ci.yml
@ -268,6 +268,7 @@ clustertests:
    - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"'
  script:
    - make clustertests
+    - make authclustertests

 external lookup tests:
  stage: integration
--- a/8
+++ b/8
@ -158,6 +158,14 @@ clustertests: vendor
 	PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1
 	$(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down

+# Run the cluster tests with authentication enabled 
+DOCKER_COMPOSE_AUTH = docker-compose -p authclustertests
+authclustertests: vendor
+	$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml down
+	$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml build
+	$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3
+	$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml run client1
+	$(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml down

 # Install Pilosa
 install:
--- a/api.go
+++ b/api.go
@ -247,7 +247,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index
 	}

 	// Create index.
-	index, err := api.holder.CreateIndexAndBroadcast(cim)
+	index, err := api.holder.CreateIndexAndBroadcast(ctx, cim)
 	if err != nil {
 		return nil, errors.Wrap(err, "creating index")
 	}
--- a/client.go
+++ b/client.go
@ -69,6 +69,7 @@ type InternalClient interface {
 	IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error

 	IDAllocDataReader(ctx context.Context) (io.ReadCloser, error)
+	IDAllocDataWriter(ctx context.Context, f io.Reader, primary *topology.Node) error
 	IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error)
 	FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error)

@ -223,6 +224,10 @@ func (n nopInternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser
 	return nil, nil
 }

+func (n nopInternalClient) IDAllocDataWriter(cctx context.Context, f io.Reader, primary *topology.Node) error {
+	return nil
+}
+
 func (n nopInternalClient) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) {
 	return nil, nil
 }
--- a/client/client.go
+++ b/client/client.go
@ -61,6 +61,8 @@ type Client struct {
 	shardNodes shardNodes
 	tick       *time.Ticker
 	done       chan struct{}
+
+	AuthToken string
 }

 func (c *Client) getURIsForShard(index string, shard uint64) ([]*pnet.URI, error) {
@ -283,7 +285,7 @@ func (c *Client) Query(query PQLQuery, options ...interface{}) (*QueryResponse,
 		return nil, errors.Wrap(err, "making request data")
 	}
 	path := fmt.Sprintf("/index/%s/query", query.Index().name)
-	_, respData, err := c.HTTPRequest("POST", path, reqData, defaultProtobufHeaders())
+	_, respData, err := c.HTTPRequest("POST", path, reqData, c.augmentHeaders(defaultProtobufHeaders()))
 	if err != nil {
 		return nil, err
 	}
@ -306,7 +308,7 @@ func (c *Client) CreateIndex(index *Index) error {

 	data := []byte(index.options.String())
 	path := fmt.Sprintf("/index/%s", index.name)
-	status, body, err := c.HTTPRequest("POST", path, data, nil)
+	status, body, err := c.HTTPRequest("POST", path, data, c.augmentHeaders(nil))
 	if err != nil {
 		return errors.Wrapf(err, "creating index: %s", index.name)
 	}
@ -330,7 +332,7 @@ func (c *Client) CreateField(field *Field) error {

 	data := []byte(field.options.String())
 	path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name)
-	status, body, err := c.HTTPRequest("POST", path, data, nil)
+	status, body, err := c.HTTPRequest("POST", path, data, c.augmentHeaders(nil))
 	if err != nil {
 		return errors.Wrapf(err, "creating field: %s in index: %s", field.name, field.index.name)
 	}
@ -398,7 +400,7 @@ func (c *Client) DeleteIndexByName(index string) error {
 	defer span.Finish()

 	path := fmt.Sprintf("/index/%s", index)
-	_, _, err := c.HTTPRequest("DELETE", path, nil, nil)
+	_, _, err := c.HTTPRequest("DELETE", path, nil, c.augmentHeaders(nil))
 	return err
 }

@ -408,7 +410,7 @@ func (c *Client) DeleteField(field *Field) error {
 	defer span.Finish()

 	path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name)
-	_, _, err := c.HTTPRequest("DELETE", path, nil, nil)
+	_, _, err := c.HTTPRequest("DELETE", path, nil, c.augmentHeaders(nil))
 	return err
 }

@ -597,7 +599,7 @@ func (c *Client) fetchFragmentNodes(indexName string, shard uint64) ([]fragmentN
 		return []fragmentNode{*c.manualFragmentNode}, nil
 	}
 	path := fmt.Sprintf("/internal/fragment/nodes?shard=%d&index=%s", shard, indexName)
-	_, body, err := c.HTTPRequest("GET", path, []byte{}, nil)
+	_, body, err := c.HTTPRequest("GET", path, []byte{}, c.augmentHeaders(nil))
 	if err != nil {
 		return nil, err
 	}
@ -635,7 +637,7 @@ func (c *Client) fetchPrimaryNode() (fragmentNode, error) {
 }

 func (c *Client) importData(uri *pnet.URI, path string, data []byte) error {
-	if status, _, err := c.doRequest(uri, "POST", path, defaultProtobufHeaders(), data); err != nil {
+	if status, _, err := c.doRequest(uri, "POST", path, c.augmentHeaders(defaultProtobufHeaders()), data); err != nil {
 		return errors.Wrapf(err, "import to %s", uri.HostPort())
 	} else if status == http.StatusPreconditionFailed {
 		return ErrPreconditionFailed
@ -683,7 +685,8 @@ func (c *Client) importRoaringBitmap(uri *pnet.URI, field *Field, shard uint64,
 		return err
 	}

-	status, _, err := c.doRequest(uri, "POST", path, defaultProtobufHeaders(), data)
+	header := c.augmentHeaders(defaultProtobufHeaders())
+	status, _, err := c.doRequest(uri, "POST", path, header, data)
 	if err != nil {
 		return errors.Wrapf(err, "roaring import to %s, status: %d", uri.HostPort(), status)
 	}
@ -724,7 +727,7 @@ func (c *Client) Info() (Info, error) {
 	span := c.tracer.StartSpan("Client.Info")
 	defer span.Finish()

-	_, data, err := c.HTTPRequest("GET", "/info", nil, nil)
+	_, data, err := c.HTTPRequest("GET", "/info", nil, c.augmentHeaders(nil))
 	if err != nil {
 		return Info{}, errors.Wrap(err, "requesting /info")
 	}
@ -754,7 +757,7 @@ func (c *Client) Status() (Status, error) {
 }

 func (c *Client) readSchema() ([]SchemaIndex, error) {
-	_, data, err := c.HTTPRequest("GET", "/schema", nil, nil)
+	_, data, err := c.HTTPRequest("GET", "/schema", nil, c.augmentHeaders(nil))
 	if err != nil {
 		return nil, errors.Wrap(err, "requesting /schema")
 	}
@ -1021,6 +1024,9 @@ func (c *Client) augmentHeaders(headers map[string]string) map[string]string {
 	version := strings.TrimPrefix(Version, "v")

 	headers["User-Agent"] = fmt.Sprintf("pilosa/client/%s", version)
+	if c.AuthToken != "" {
+		headers["Authorization"] = c.AuthToken
+	}
 	return headers
 }

@ -1177,7 +1183,7 @@ func (c *Client) startTransaction(id string, timeout time.Duration, exclusive bo
 		return nil, errors.Wrap(err, "marshalling transaction")
 	}

-	status, data, err := c.httpRequest("POST", "/transaction", bod, defaultJSONHeaders(), true)
+	status, data, err := c.httpRequest("POST", "/transaction", bod, c.augmentHeaders(defaultJSONHeaders()), true)
 	if status == http.StatusConflict && time.Now().Before(deadline) {
 		// if we're getting StatusConflict after all the usual timeouts/retries, keep retrying until the deadline
 		time.Sleep(time.Second)
@ -1204,7 +1210,7 @@ func (c *Client) startTransaction(id string, timeout time.Duration, exclusive bo
 }

 func (c *Client) FinishTransaction(id string) (*pilosa.Transaction, error) {
-	_, data, err := c.httpRequest("POST", "/transaction/"+id+"/finish", nil, defaultJSONHeaders(), true)
+	_, data, err := c.httpRequest("POST", "/transaction/"+id+"/finish", nil, c.augmentHeaders(defaultJSONHeaders()), true)
 	if err != nil && len(data) == 0 {
 		return nil, err
 	}
@ -1226,7 +1232,7 @@ func (c *Client) FinishTransaction(id string) (*pilosa.Transaction, error) {
 }

 func (c *Client) Transactions() (map[string]*pilosa.Transaction, error) {
-	_, respData, err := c.httpRequest("GET", "/transactions", nil, defaultJSONHeaders(), true)
+	_, respData, err := c.httpRequest("GET", "/transactions", nil, c.augmentHeaders(defaultJSONHeaders()), true)
 	if err != nil {
 		return nil, errors.Wrap(err, "getting transactions")
 	}
@ -1240,7 +1246,7 @@ func (c *Client) Transactions() (map[string]*pilosa.Transaction, error) {
 }

 func (c *Client) GetTransaction(id string) (*pilosa.Transaction, error) {
-	_, data, err := c.httpRequest("GET", "/transaction/"+id, nil, defaultJSONHeaders(), true)
+	_, data, err := c.httpRequest("GET", "/transaction/"+id, nil, c.augmentHeaders(defaultJSONHeaders()), true)
 	if err != nil {
 		return nil, err
 	}
--- a/cmd/backup.go
+++ b/cmd/backup.go
@ -31,5 +31,6 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
 	flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
 	flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
 	ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
+	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
 	return ccmd
 }
--- a/cmd/import.go
+++ b/cmd/import.go
@ -5,7 +5,7 @@ import (
 	"context"
 	"io"

-	"github.com/molecula/featurebase/v3"
+	pilosa "github.com/molecula/featurebase/v3"
 	"github.com/molecula/featurebase/v3/ctl"
 	"github.com/spf13/cobra"
 )
@ -51,6 +51,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM.
 	flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.")
 	flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.")
 	ctl.SetTLSConfig(flags, "", &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification)
+	flags.StringVar(&Importer.AuthToken, "auth-token", "", "Authentication token")

 	return importCmd
 }
--- a/cmd/restore.go
+++ b/cmd/restore.go
@ -27,6 +27,7 @@ The Restore command will take a backup archive and restore it to a new, clean cl
 	flags.IntVar(&cmd.Concurrency, "concurrency", 1, "number of concurrent uploads")
 	flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.")
 	flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.")
+	flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token")
 	ctl.SetTLSConfig(
 		flags, "",
 		&cmd.TLS.CertificatePath,
--- a/ctl/backup.go
+++ b/ctl/backup.go
@ -52,6 +52,8 @@ type BackupCommand struct { // nolint: maligned
 	*pilosa.CmdIO

 	TLS server.TLSConfig
+
+	AuthToken string
 }

 // NewBackupCommand returns a new instance of BackupCommand.
@ -93,6 +95,10 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
 	}
 	cmd.client = client

+	if cmd.AuthToken != "" {
+		ctx = context.WithValue(ctx, "token", "Bearer "+cmd.AuthToken)
+	}
+
 	// Determine the field type in order to correctly handle the input data.
 	indexes, err := cmd.client.Schema(ctx)
 	if err != nil {
--- a/ctl/import.go
+++ b/ctl/import.go
@ -54,6 +54,8 @@ type ImportCommand struct { // nolint: maligned
 	*pilosa.CmdIO

 	TLS server.TLSConfig
+
+	AuthToken string
 }

 // NewImportCommand returns a new instance of ImportCommand.
@ -84,6 +86,10 @@ func (cmd *ImportCommand) Run(ctx context.Context) error {
 	}
 	cmd.client = client

+	if cmd.AuthToken != "" {
+		ctx = context.WithValue(ctx, "token", "Bearer "+cmd.AuthToken)
+	}
+
 	if cmd.CreateSchema {
 		if cmd.FieldOptions.Type == "" {
 			// set the correct type for the field
--- a/ctl/import_test.go
+++ b/ctl/import_test.go
@ -9,12 +9,17 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"os"
 	"reflect"
 	"strings"
 	"testing"
 	"time"

-	"github.com/molecula/featurebase/v3"
+	"github.com/golang-jwt/jwt"
+	pilosa "github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/authn"
+	"github.com/molecula/featurebase/v3/logger"
+	"github.com/molecula/featurebase/v3/server"
 	"github.com/molecula/featurebase/v3/test"
 	"github.com/molecula/featurebase/v3/testhook"
 )
@ -568,3 +573,156 @@ func TestImportCommand_RunBool(t *testing.T) {
 		}
 	})
 }
+
+func TestImport_AuthOn(t *testing.T) {
+	clusterSize := 1
+
+	logFilename := "./testdata/query.log"
+	_, err := os.Create(logFilename)
+	if err != nil {
+		t.Fatalf("Failed to create query log file: %s", err)
+	}
+
+	auth := server.Auth{
+		Enable:           true,
+		ClientId:         "e9088663-eb08-41d7-8f65-efb5f54bbb71",
+		ClientSecret:     "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
+		AuthorizeURL:     "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize",
+		TokenURL:         "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token",
+		GroupEndpointURL: "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true",
+		LogoutURL:        "https://login.microsoftonline.com/common/oauth2/v2.0/logout",
+		Scopes:           []string{"https://graph.microsoft.com/.default", "offline_access"},
+		SecretKey:        "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF",
+		RedirectBaseURL:  "https://localhost:0",
+		QueryLogPath:     logFilename,
+		PermissionsFile:  "./testdata/permissions.yaml",
+	}
+
+	commandOpts := make([][]server.CommandOption, clusterSize)
+	configs := make([]*server.Config, clusterSize)
+	for i := range configs {
+		conf := server.NewConfig()
+		configs[i] = conf
+		conf.Bind = "https://localhost:0"
+		conf.Auth = auth
+		conf.TLS.CertificatePath = "./testdata/certs/localhost.crt"
+		conf.TLS.CertificateKeyPath = "./testdata/certs/localhost.key"
+		conf.TLS.CACertPath = "./testdata/certs/pilosa-ca.crt"
+		conf.TLS.EnableClientVerification = false
+		conf.TLS.SkipVerify = true
+		commandOpts[i] = append(commandOpts[i], server.OptCommandConfig(conf))
+	}
+
+	a, err := authn.NewAuth(
+		logger.NewStandardLogger(os.Stdout),
+		"http://localhost:0/",
+		auth.Scopes,
+		auth.AuthorizeURL,
+		auth.TokenURL,
+		auth.GroupEndpointURL,
+		auth.LogoutURL,
+		auth.ClientId,
+		auth.ClientSecret,
+		auth.SecretKey,
+	)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// make a valid token
+	tkn := jwt.New(jwt.SigningMethodHS256)
+	claims := tkn.Claims.(jwt.MapClaims)
+	groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "group-id-test", GroupName: "group-name-test"}})
+	claims["molecula-idp-groups"] = groupString
+	claims["oid"] = "42"
+	claims["name"] = "valid"
+	token, err := tkn.SignedString([]byte(a.SecretKey()))
+	if err != nil {
+		t.Fatal(err)
+	}
+	validToken := "Bearer " + token
+	invalidToken := "Bearer " + string(tkn.Raw)
+
+	tests := []struct {
+		Index        string
+		Field        string
+		CreateSchema bool
+		Token        string
+		Err          error
+	}{
+		{
+			Index:        "test",
+			Field:        "field1",
+			CreateSchema: true,
+			Token:        validToken,
+			Err:          nil,
+		},
+		{
+			Index:        "test",
+			Field:        "field1",
+			CreateSchema: false,
+			Token:        validToken,
+			Err:          nil,
+		},
+		{
+			Index:        "test",
+			Field:        "field1",
+			CreateSchema: false,
+			Token:        invalidToken,
+			Err:          fmt.Errorf("token contains an invalid number of segments"),
+		},
+		{
+			Index:        "test",
+			Field:        "field1",
+			CreateSchema: true,
+			Token:        invalidToken,
+			Err:          fmt.Errorf("token contains an invalid number of segments"),
+		},
+	}
+
+	t.Run("set", func(t *testing.T) {
+		buf := bytes.Buffer{}
+		stdin, stdout, stderr := GetIO(buf)
+		cm := NewImportCommand(stdin, stdout, stderr)
+		file, err := testhook.TempFile(t, "import.csv")
+		if err != nil {
+			t.Fatalf("creating tempfile: %v", err)
+		}
+		_, err = file.Write([]byte("1,2\n3,4\n5,6"))
+		if err != nil {
+			t.Fatalf("writing to tempfile: %v", err)
+		}
+
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		cluster := test.MustRunCluster(t, clusterSize, commandOpts...)
+		defer cluster.Close()
+		cmd := cluster.GetNode(0)
+		cm.Host = cmd.API.Node().URI.HostPort()
+
+		for i, test := range tests {
+			cm.Index = test.Index
+			cm.Field = test.Field
+			cm.CreateSchema = test.CreateSchema
+			cm.Paths = []string{file.Name()}
+			ctx := context.WithValue(context.Background(), "token", test.Token)
+			err = cm.Run(ctx)
+			if test.Err != nil {
+				if !strings.Contains(err.Error(), test.Err.Error()) {
+					t.Fatalf("Test: %d, Import Run doesn't work: got %s, expected: %s", i, err, test.Err)
+				}
+			} else {
+				if err != test.Err {
+					t.Fatalf("Test: %d, Import Run doesn't work: got %s, expected: %s", i, err, test.Err)
+				}
+			}
+
+		}
+		err = os.Remove(logFilename)
+		if err != nil {
+			t.Fatalf("Failed to delete query log file: %s", err)
+		}
+	})
+}
--- a/ctl/restore.go
+++ b/ctl/restore.go
@ -29,7 +29,8 @@ import (
 // RestoreCommand represents a command for restoring a backup to
 type RestoreCommand struct {
 	tlsConfig *tls.Config
-	Host      string
+
+	Host string

 	Concurrency int

@ -47,7 +48,10 @@ type RestoreCommand struct {

 	// Standard input/output
 	*pilosa.CmdIO
+
 	TLS server.TLSConfig
+
+	AuthToken string
 }

 // NewRestoreCommand returns a new instance of RestoreCommand.
@ -88,6 +92,10 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
 	}
 	cmd.client = client

+	if cmd.AuthToken != "" {
+		ctx = context.WithValue(ctx, "token", "Bearer "+cmd.AuthToken)
+	}
+
 	nodes, err := cmd.client.Nodes(ctx)
 	if err != nil {
 		return err
@ -139,8 +147,24 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology.
 	if len(existingSchema) == 0 {
 		cmd.Logger().Printf("Load Schema")
 		url := primary.URI.Path("/schema")
+		req, err := retryablehttp.NewRequest("POST", url, f)
+		if err != nil {
+			return err
+		}
+		req = req.WithContext(ctx)
+		req.Header.Add("Accept", "application/json")
+
+		token, ok := ctx.Value("token").(string)
+		if ok && token != "" {
+			req.Header.Set("Authorization", token)
+		}
+
 		client := cmd.newClient()
-		_, err = client.Post(url, "application/json", f)
+		_, err = client.Do(req)
+		if err != nil {
+			return err
+		}
+
 	} else {
 		schema := &pilosa.Schema{}
 		if err := json.NewDecoder(f).Decode(schema); err != nil {
@ -222,10 +246,9 @@ func (cmd *RestoreCommand) restoreIDAlloc(ctx context.Context, primary *topology
 	defer f.Close()

 	logger.Printf("Load idalloc")
-	url := primary.URI.Path("/internal/idalloc/restore")

-	client := cmd.newClient()
-	_, err = client.Post(url, "application/octet-stream", f)
+	err = cmd.client.IDAllocDataWriter(ctx, f, primary)
+
 	return err
 }

@ -301,6 +324,11 @@ func (cmd *RestoreCommand) restoreShard(ctx context.Context, filename string) er
 		req = req.WithContext(ctx)
 		req.Header.Set("Content-Type", "application/octet-stream")

+		token, ok := ctx.Value("token").(string)
+		if ok && token != "" {
+			req.Header.Set("Authorization", token)
+		}
+
 		client := cmd.newClient()
 		resp, err := client.Do(req)
 		if err != nil {
--- a/ctl/testdata/certs/README.md
+++ b/ctl/testdata/certs/README.md
@ -0,0 +1,12 @@
+
+
+# these test certs were generated with the following commands
+
+certstrap --depot-path certs init --common-name pilosa-ca --expires "100 years"
+certstrap --depot-path certs request-cert --common-name localhost --domain localhost
+certstrap --depot-path certs sign "localhost" --CA pilosa-ca --expires "100 years"
+
+# certstrap version
+dev-25ea708a
+
+(built with go 1.13)
--- a/ctl/testdata/certs/localhost.crt
+++ b/ctl/testdata/certs/localhost.crt
@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEPjCCAiagAwIBAgIRAJ7rl74WPv8pLuhVRXt6fV0wDQYJKoZIhvcNAQELBQAw
+FDESMBAGA1UEAxMJcGlsb3NhLWNhMCAXDTIwMTAyMDE5MTMzNFoYDzIxMjAxMDIw
+MTkxMzE5WjAUMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDmi8FMWt23M0Cr2aCgEXUGQ0gv/4M7CXH/5GkSI866YwGV
+Bd1iZMBRiONQwvGDnqYZRrAQv6mFjfyBqxdkbh++74FC3JK7sLhks0vg5VwbHV7T
+5kj3bJqd+LKn5qPPOQXX9sgmv/NkggF/XXwF73noLPmgDQ78S+OP0ANmi1TQiU3a
+gE+qp+Qpl5KC7dH9aC9nvE9iGfEcGNr+rXj05liiXqe4ZtIKWjeke7Ej64C6qX97
+bNPzmLARtqbRsIkfAU8SJy3YuHfW8n1xr4B7ENm9jHQCh1wUv2YhaPpnio+/R2zp
+Lw4yCqilDX9ZZ4nG3cBFuziSf+BUXJ9ydbw1aX8HAgMBAAGjgYgwgYUwDgYDVR0P
+AQH/BAQDAgO4MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAdBgNVHQ4E
+FgQUJRQJpaR5bp4ZyUsMxgK+UJl6PSgwHwYDVR0jBBgwFoAU69lmXSa5BZeyYU/6
+XpWdtr59H1YwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IC
+AQChxsBZ/b14ukJXX48BxAyZcy5r7GrLcRGQ3guUTONFVDWPzzpd8mjHi0yJDhMW
+2zWtw3/H+c+zT7uRd+2sUxFdpAurNSFCdV++5Q/0aFvl+By5+MhVhtznEQDU0/lM
+zFxiEYe/N9Vi2N0S1KPxvYL/RfBU27u+O/50zhjueM1BTyHTTqL6E2DFeT2VPKIg
+zCDUtiTEDFZrD0XGITT/3CIoNCK8aC+Fq65OEoyEn6qR5qg1Kc4tfZmo6hWYiSlR
+XeP36cP9R8kEMte1BdE74GVqE9cTuVZERdgB0hv3EME7Byq7uIm/a+JXbsh2/OFm
+HcE0/HP+O0YK8YaVMGwI3pZYy2syWqPcakcvusETehr6P+Ihh2cOKRwqkCl6b87e
+uSLJNTUMKZgakW6Bjv6lgQaWqnKzTC/RgmQ+G3w0nKATX9+jYE2j3MzZhbtcml+2
+gp6u225yAJaYt/MQidwUMiKYeCgjaUNoL0fOJesGkokPk80ceISnqvbSRiZRTvK1
+bVenkhkBrHuvvgKVstzcuZI9oQ2snWhK1naVQiOtQNEFUCHwyU95zADOK0km88NB
+2het6yYaEUL9csHPEjPd3lFglerGQnil2Ly1slUC4jb7hfVRHjOFs8PVr9gQ45dW
+Jvsv4pawHKFE0ennoNvoDmzbiY1TY5ScTZquPGIsEBV+tQ==
+-----END CERTIFICATE-----
--- a/ctl/testdata/certs/localhost.csr
+++ b/ctl/testdata/certs/localhost.csr
@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICgDCCAWgCAQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEA5ovBTFrdtzNAq9mgoBF1BkNIL/+DOwlx/+RpEiPO
+umMBlQXdYmTAUYjjUMLxg56mGUawEL+phY38gasXZG4fvu+BQtySu7C4ZLNL4OVc
+Gx1e0+ZI92yanfiyp+ajzzkF1/bIJr/zZIIBf118Be956Cz5oA0O/Evjj9ADZotU
+0IlN2oBPqqfkKZeSgu3R/WgvZ7xPYhnxHBja/q149OZYol6nuGbSClo3pHuxI+uA
+uql/e2zT85iwEbam0bCJHwFPEict2Lh31vJ9ca+AexDZvYx0AodcFL9mIWj6Z4qP
+v0ds6S8OMgqopQ1/WWeJxt3ARbs4kn/gVFyfcnW8NWl/BwIDAQABoCcwJQYJKoZI
+hvcNAQkOMRgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQAD
+ggEBABMi2/4j1/qzwWAYlEs2KW3z+apzzDLKgjE0kY6QvELh/8aBj0rMglb0HM2x
+4iSSoX1ZwZgDZ9fIJ3klG/UF7CUweMghb9yC2PP9Z8WuqaECQyM87KgSln8PND9E
+1OvD30rp9yr9KxEeckq+c1ebLi/qGrIY21VCwfxA0mv3sfi7Q5ONIckay/Xj+1Tz
+ovE/TkM/8wTE/SKbpQSCkP7K1NDXuAhMGjcN0x3d3f8nBcLcZOrRroiHy38Bv/9T
+Vd62IY6uqYw9sluBbMX72D/mmJiCKEw3+DhDJFhHCTCrAQM0QwLuwnG2lQFYoENc
+ZAkwDIi+3DXHEEyloNSYGtXMEiA=
+-----END CERTIFICATE REQUEST-----
--- a/ctl/testdata/certs/localhost.key
+++ b/ctl/testdata/certs/localhost.key
@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA5ovBTFrdtzNAq9mgoBF1BkNIL/+DOwlx/+RpEiPOumMBlQXd
+YmTAUYjjUMLxg56mGUawEL+phY38gasXZG4fvu+BQtySu7C4ZLNL4OVcGx1e0+ZI
+92yanfiyp+ajzzkF1/bIJr/zZIIBf118Be956Cz5oA0O/Evjj9ADZotU0IlN2oBP
+qqfkKZeSgu3R/WgvZ7xPYhnxHBja/q149OZYol6nuGbSClo3pHuxI+uAuql/e2zT
+85iwEbam0bCJHwFPEict2Lh31vJ9ca+AexDZvYx0AodcFL9mIWj6Z4qPv0ds6S8O
+MgqopQ1/WWeJxt3ARbs4kn/gVFyfcnW8NWl/BwIDAQABAoIBAFX+GPqfBgY4cs3m
+3ff2qvzMCdgFaXCS5Fe7XcmrW4fAOC3awynZRLbk5U0Reb5LZc8Vw8RriRLM1DuV
+kqMeRG8WrNNArOafUxgUnJ/lTUa73MwTIHJRqxZzVkg0SjOYJGranOt/O4zoxSA5
+wXIBUipc5Dtjw4wtzlKtFyefnuItL2MCdwOHUdZfnhr9Oykp1fuNqBqkkeryj3XV
+ukHQvqU5zkMSayprNglziqTHUzU33iyZeDng+CJQeYTEc7Gn+zja2SFFBlPHqXXo
+/OzAr94zI3vOnj3yRM3+sKMJVPV+RoJEGpsvPVuVn38d1VnIMEx8Gy/wif6tmM9c
+7Q44hKECgYEA/JMFwkPGbry80ktDI065k5FIYn1EDRyUaQqyskmkBRcNW3qOShqj
+o/zWQfCgxP587IEdKBBwqCpdqfghi3EW+JqfVlbGY6t1chAurYF/47CTIgKO5qRM
+GdCY2OdiAeo5nba/KiLQfSuY08MCNDrQabLRJXIng8qVWpRwQzsv5rECgYEA6aw/
+HugeQhTxk2uV91jJaAQIaxrt6JxuoG0CGGlbDrrTl2dnbPYA0muHMFdT/bzKjCpv
+n/ScqbCyHuy+lWSnOzgedRNQCB46+0H58LAjITAj9QaT3raZqVReVIaD+pnx27dp
+Cw5Ws6ENa9AQey3DO+dkRWot2AcLSw6TGR8HnzcCgYEA/ArfCU/G2cSgDJ6sPbSW
+vaqR+C6W1Rq7AuN5FS8lbSrm2m2/RjW1LLTnPmAYntxx3zSs2sklErtMQovpNZRB
+3w21iVwIl3eHOK7rVZtP+u++s4aoAYLcqjod/P1RMSYCHt85fpvFP9Ncq50DOwmh
+5ohZ6ysyQXLMfdp4+K48i9ECgYEApOFALKu2ZgRnLRFV4REKFFX8Jq76vg5bVOF2
+AAmfEbasBIIXDWBL1i2/V1HXVwv2k46B8wjj3ixqkr2UAM/j3DpN62g0KXZDQfUc
+ykNOlmVkickZX6XSqRN5+ARubc5gRRuWiBGXBeqXEMLgTjpNLyCntP8l1++ofU6M
+ZsZpV2MCgYA3nfNXAR5O4B/dm/2HmDQrXy0qia7Hwi/95pgL2FJaEmjBCPI1j12o
+M5YCbhpr1pwsNKPV9AUlUz+OCwS8Vt+V0gQf9/XvNOsifU+mbMYVpuNGwmcKafnv
+qECSeidrmhWJSR/SSNBcE94im/8ObVU110WJMkjC9otjDl9Aua/3LQ==
+-----END RSA PRIVATE KEY-----
--- a/ctl/testdata/certs/pilosa-ca.crl
+++ b/ctl/testdata/certs/pilosa-ca.crl
@ -0,0 +1,16 @@
+-----BEGIN X509 CRL-----
+MIIChTBvAgEBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMTCXBpbG9zYS1jYRcN
+MjAxMDIwMTkxMzIyWhgPMjEyMDEwMjAxOTEzMjBaMACgIzAhMB8GA1UdIwQYMBaA
+FOvZZl0muQWXsmFP+l6Vnba+fR9WMA0GCSqGSIb3DQEBCwUAA4ICAQBja+EDQAp+
+KeD7UhWMMrTd9j03GgQ2E2Z7+Ba0qJ5+kS7/t+Yja2o5dQJkrC3GwEMOQb6DRRUE
+nUE4xlr5Rryoq0dZk+Lp1f4cHrnP8l1xylUL44gsnY4v8zMR8L8X98vj7kKCqB8w
+DFX7qkMlE5Ie2Hha7uuOJ85FnxIbMcRxFQH2m2zDfWG8/Lmxezvv9Hn45V/kwIQy
+MmBh6cNuhzEneyNpM9yMRe/29QgVitF/2q6d+FzK8w8hkUFeYlyM+cP7F4Ml7160
+UidSQM04zvBtJ8frZAvrDaPBBZhrTXcyw6+Qnp/aaW1ZsEIdHEcbYGNdgtazleoG
+VH35cDP90KfiRbq69PQ9Zqn3cI//MX3sHrglA9wsEhHc9P7dowHaOFyxPouZPEmQ
+/Jqg5oyJzujRwhf0v3SdJvhuDEzla2N+QyYRk0kRHtdv+glz7T7CnTYCk+DTv+oh
+QABUrCbjfBoE5M2Qep9ZkIbl2gaDCpvbZSF4zFLKQc2aIOBpVn3HgTGBvdFD3FJY
+Txl2F4Y3rS1T/WMAH86cZIc9h5HlMdFtAFnHAlHtB3wGw3FD/GcvGcvz2D4GaxKq
+erzrnOxjYOA4M0haGzWF6dC7aPA8y35eZuqNXvbenTtc7A11bWTJfG1I7ctvLyPE
+MpCNMHfymh/XtYZiZhvu6ueu3OeKScN+tA==
+-----END X509 CRL-----
--- a/ctl/testdata/certs/pilosa-ca.crt
+++ b/ctl/testdata/certs/pilosa-ca.crt
@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE6jCCAtKgAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwlwaWxv
+c2EtY2EwIBcNMjAxMDIwMTkxMzIyWhgPMjEyMDEwMjAxOTEzMjBaMBQxEjAQBgNV
+BAMTCXBpbG9zYS1jYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALT/
+uNmbnfXWNX+FsL0Waqw/5deti5F4cSjMrGRQpXxalTcooqNk/lkeqXkvi9ooFROZ
+/HyQR9GM9dSD/aj6gD3FnGA4ueB24Xr6bWsRpDRh6+3UGLB3YCNNdGLSfX3LPMYh
+RJutFmsg+r6SrSytbLbffu+0a/4fxtajZNwQJjDjd8qflXQZYlzp2LHk1A/jqqdI
+fBtqkNg925TGKiavvUqKtdI/eFzRoiQ7NLBUJmszzveUXvOUMsMnW2/myLBe3Oqk
+Vsy85lya0ADln20C3Lb0+ZA4KoGX3EWdtBXEuWqMoyvCJoJ4I3bH2LlfOUjRt8UE
+pPk6sPMROJ+75mlgvgnSlYsN8PaZdvdm2VGVWRWUyEfyW/qa2fv8d2XBWqibl0YF
+tqay9CX1aWgC9q12yx3vj7Yh+ZNbeZFLc7IL8zyNMwIjIOIIyGBY70KewfgVktzq
+fAMz6h1sr9Kxozil97Cu3ma4B6UiL3rUbYMO/rNhVxcIuUoJpgIEVuRt+uXEG74y
+XftauZ67qILFQzfpoacncvDEx5nJ3itLgbbyt1n1iWdGuEiMSLFT+x+nNUgVpQgA
+sWRYxHdisM4xzRVN6pAaToMs1p8Ju7l9xU3z7RSogTyVk9gMIIV4t9TDYP5fI10Y
+GFi7B6q0t3pIGXgKySHjCSl0EKYkDQDFl5tWeaiXAgMBAAGjRTBDMA4GA1UdDwEB
+/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTr2WZdJrkFl7Jh
+T/pelZ22vn0fVjANBgkqhkiG9w0BAQsFAAOCAgEAnlBFrWhB+WesCc3lhK980rA6
+roNFYMZdaXvg4zaEGergkRvPab5yXoof1AAeznJm45GQfXn8HbQlrZmAqWg3fNld
+/TX+jNvosM8K8K+PzesDGHsm/eQnbrb0qzMDsQgFY+nnD+x/ZQtjmKZtcNr/0ZlM
+EJeXWU5cGy70GMbNztspMHsOLa3ZDLsBOJYOwSFxDlLDFrjZoRoPCWw8jRL+Tb4t
+JjZcGZDD4a5+DqcojanIdNU1yI4teP6aV1LQTVNn4pwOap+tD0De/WzOPmXTQq5M
+9ssxL7xSqVShQQMC8LVSWSRxtT6kLq0Av6i7wio0DZGnH3ynERTUs13DRZkwbVsE
+OaQLmiQnsHRTIpdts/fswZ2FRPvdhhXxBjiGQZGEGXXznxHTNJ6nQioh95Ft5hNA
+82i8Z74miaFIT/33/sZ5SuwUzphCgqCY2x7NUS8J313O9lsar0bweJTvQaZg/69E
+PmmwUcDebh+pgKP01z4BqTzhtchmFUKzT+oOC8tmTeSlhsBTzO4xMw6OqhpXKL+c
+k9f2CGUZYtEZHDRmP+C++FEi+B/tV2Oq3on+QPiaIIcRsOftthGUvJ8htUl3w+hq
+B5TnL8CeLjXGKKRp+UiakrB4E7y2aIbrtIRnJ/Llg2XMND/0xbldsNsyDNXCIoDH
+sz8HqwF3CUbv5XD4ioY=
+-----END CERTIFICATE-----
--- a/ctl/testdata/certs/pilosa-ca.key
+++ b/ctl/testdata/certs/pilosa-ca.key
@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAtP+42Zud9dY1f4WwvRZqrD/l162LkXhxKMysZFClfFqVNyii
+o2T+WR6peS+L2igVE5n8fJBH0Yz11IP9qPqAPcWcYDi54HbhevptaxGkNGHr7dQY
+sHdgI010YtJ9fcs8xiFEm60WayD6vpKtLK1stt9+77Rr/h/G1qNk3BAmMON3yp+V
+dBliXOnYseTUD+Oqp0h8G2qQ2D3blMYqJq+9Soq10j94XNGiJDs0sFQmazPO95Re
+85Qywydbb+bIsF7c6qRWzLzmXJrQAOWfbQLctvT5kDgqgZfcRZ20FcS5aoyjK8Im
+gngjdsfYuV85SNG3xQSk+Tqw8xE4n7vmaWC+CdKViw3w9pl292bZUZVZFZTIR/Jb
+prZ+/x3ZcFaqJuXRgW2prL0JfVpaAL2rXbLHe+PtiH5k1t5kUtzsgvzPI0zAiMg
+4gjIYFjvQp7B+BWS3Op8AzPqHWyv0rGjOKX3sK7eZrgHpSIvetRtgw7+s2FXFwi5
+SgmmAgRW5G365cQbvjJd+1q5nruogsVDN+mhpydy8MTHmcneK0uBtvK3WfWJZ0a4
+SIxIsVP7H6c1SBWlCACxZFjEd2KwzjHNFU3qkBpOgyzWnwm7uX3FTfPtFKiBPJWT
+2AwghXi31MNg/l8jXRgYWLsHqrS3ekgZeArJIeMJKXQQpiQNAMWXm1Z5qJcCAwEA
+AQKCAgAWmjiDNCOtp2pW2mMPudToXbJeFJXxPJEk/yon/MotlUI8+R4WOW5pwqJ3
+N7DHNWosYHZfN8VALdIlD7aFe4K4NA0rFupfVXki2lL/o9xVjkTgFjRfFQk0X1/B
+V3fEVbTpKQ5gQmUiS6QEWFy3z5Bb5dz8IhO6UE2MUCswL/QU9tLmwrbvIJxf7fPZ
+gzHYKh4NdcfJxK0B0/evxG9PFXMV8+xwrOxi6urMi3gw7NE/YeDemfChikgshqWs
+e61kGPSNeKg+OPirZ8nB0urtugXF8yGXGOx18njXWLI8Zayh2Z4mwL/+WvJSyvIN
+dA67QTUprULMvL+MGwJvMA+96Q7SBRVKR9HHNaP9pFsZup3QX9mqDQ9miB6+rzn7
+f5RiSLVgq+HUPMPfqgXCkQZBcY28TcM1BZhS4uJJTkbvVTkrlHKJs6Q/LBqJFvq0
+3+2M1xQb4HdRTRlwZ/YsxdqXGIoA3Xx3nZbb6LlPp/MT93xxsLJNNP47n445Cw8i
+lz7hJJDwo+TyXmRRWKlFXO8TEhqKhK9ZEXmkXBxSeCQV0oTYS5kU9XdZ5iu5/CQQ
+Lv+uFQfHTPWm/Lp5RC8JEEJwK5bwRAs5d9oWg5EbWJp2ol36g3YO1np5RFsQn3jL
+qJPz35X3Bp9zQeZcAZpt1fWFdyX9f7V2LLCY8gUyIlfCEXBWwQKCAQEA8Uoes4ph
+15tM1LswmOLqOq9iWuvcKYSNcz8my8nlP5zkUdGKvGcLblm5Mg4wKoPFiyjKhX7I
+S8DUN8x7E5aiNBiZ0PGku4CKjubQgdFG/rYRfnrUEaA81Uw3a69eYioKbxhhxDzb
+Lqd0/tGNHQZBQEyofOChwqTWpAdIh79F1oXejcPzRDr+oXxJVnvJDdYCEN/0TYkz
+qeJdEtnVf1x2oNjuPIRuZNldpSDmiUce4QXG/qKgJcQNZ+paDDFK/G/rXHcIvV5C
+du9yxxfppY7fRRMm+LFqDhKEWveG4OUhUgut71J0EREO65oMbf6UcZ2FS4XDTbFD
+RSO4d8bKgf2eKwKCAQEAwAigvp8qq/yYitZeXhI6cI4ztSvwqblREhUgYWyrbUFo
+a38Bey1fKQzJYUYA7raFU6alRHoHBQywjANhIaLlfvLtQfuZ4Z9CGP3Qn33uQR/E
+ha4MNjjwUB0jx9lsDze1h61V95fxQLGNLVwGoaES4BpDRqvYJKnQv2X0SUigEg5U
+GwryNlEW0AS/Xp/k7+PGJQernHIEWYS70FleHbAiINh+lzSfbJObgd6XnQ8IxtTr
+xthXBKkkNBJdJX+/3qUQgOxTjSNUY4N9Np7myFfMvcAXuR7/K7bDegwHxffYb6Gc
+v3fCFoTQFn1KTh0IvRjyv3WzqInAYVjC8CpD562VRQKCAQBirI4LnE7Q7lioMnj4
+POvO3gRZ7FSXwfZap/vEoScYMaAJeajDzVwWX6jluHmoGUVC2IahuyxMFmpy+zNl
+2lcw+NKGaRuV9kYzlF62iBABgBF9aNuq7Z2TGN0dM5VkjY7AyfbJWp3D4YVt4+JS
+eUlb8z1//BkK0YBZigT2RplX1l0iGn00bO/OuFYBgRPCjb9AiWWOA8rV8ZVgbSbr
+M7PrqWsb4oiGw4GRUvgUMbqGCWfMoFLfvuJAmc0DaXEh9N8KbD9tuctyeg+1LalG
+JDxYMjHgyCT35kisLsfA1tMei1oxIcYHaLNyVAg7Pz4TjHiDXwt0jUZWUvpQOUJ9
+kGsLAoIBAQCOqFoyAjBDIB16VpI4NDZx01IabxAUJfVSB5vMhFw9h++4m9tP1H7z
+Eeqwdr7Ol40ofY4c9sIsQCcPfJs1z7vJuVIESJMih5sk0bmgIn9SpfTqkkfEKDxu
+Z5djKeQa0fnrVxucGaZBtyT343uRqwVIsnn0EEk7w2OuLGFz553yi+5zQIh7TXYz
+BrPb6dC7XWyfqbkVOaZ9khusRhei2mwgFnTEg3VDxcwqiF/9b2PHwfl9+M18SuL4
+RAQqjWLOVbWS8P2Ixgw0+UOVxioP/xm8hO2auqo5oUZKbpF/wgVpuJenraHj9LpZ
+Wq5OpUcOo3ACR8A1nk/qgXQf0mYrwEo5AoIBAHEqA2eJVZnPiAs6U7QPAavnLxt/
+v0GLzsBBixSV8ErMToN1wfYtBb1t5fgF0Fuy85dREp1CsGJMgrnPX5bCnBmDaLl2
+Z1lUaSDcFCu+yXo+Kuy7JvSKZ4++q4ggrHvK8y8FdKH4H+56vTdXe2i9RY/v48g4
+kKyNiYtVXxrd/h47WbHF5eApheblH9hH6zC5tB/rW7Hh0nmnDcfmMW4BggbyBinH
+MF3jO0YaspZOtRc2xSj8E3sGtN+f/KrBbKBb4J0j7VzuFmZC1u5grl/hx0cYE2ek
+HGifmIjkKv5R4xPELoAJZyFOpN1PfS3Y+SOn0mF+RJRoGqMGcQWA3I77b5M=
+-----END RSA PRIVATE KEY-----
--- a/ctl/testdata/permissions.yaml
+++ b/ctl/testdata/permissions.yaml
@ -0,0 +1,4 @@
+user-groups:
+  "group-id-test":
+    "test": "write"
+admin: "group-id-test"
--- a/holder.go
+++ b/holder.go
@ -1083,7 +1083,7 @@ func (h *Holder) LoadView(index, field, view string) (*view, error) {
 // CreateIndexAndBroadcast creates an index locally, then broadcasts the
 // creation to other nodes so they can create locally as well. An error is
 // returned if the index already exists.
-func (h *Holder) CreateIndexAndBroadcast(cim *CreateIndexMessage) (*Index, error) {
+func (h *Holder) CreateIndexAndBroadcast(ctx context.Context, cim *CreateIndexMessage) (*Index, error) {
 	h.mu.Lock()
 	defer h.mu.Unlock()

@ -1093,7 +1093,7 @@ func (h *Holder) CreateIndexAndBroadcast(cim *CreateIndexMessage) (*Index, error
 	}

 	// Create the index in etcd as the system of record.
-	if err := h.persistIndex(context.Background(), cim); err != nil {
+	if err := h.persistIndex(ctx, cim); err != nil {
 		return nil, errors.Wrap(err, "persisting index")
 	}

--- a/http/client.go
+++ b/http/client.go
@ -142,6 +142,14 @@ func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client, o
 	return ic
 }

+func AddAuthToken(ctx context.Context, req *http.Request) *http.Request {
+	token, ok := ctx.Value("token").(string)
+	if ok && token != "" {
+		req.Header.Set("Authorization", token)
+	}
+	return req
+}
+
 // MaxShardByIndex returns the number of shards on a server by index.
 func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) {
 	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.MaxShardByIndex")
@ -162,6 +170,7 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -194,6 +203,7 @@ func (c *InternalClient) AvailableShards(ctx context.Context, indexName string)

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -227,6 +237,7 @@ func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bo

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -258,6 +269,7 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -294,6 +306,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf []
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
 	if err != nil {
@ -344,6 +357,7 @@ func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, in
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
@ -376,6 +390,7 @@ func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("Accept", "application/x-protobuf")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
@ -404,6 +419,7 @@ func (c *InternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, indexNam
 	}
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
@ -434,6 +450,7 @@ func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilos
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
@ -480,6 +497,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request against the host.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -509,6 +527,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -540,6 +559,7 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) {

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -590,6 +610,8 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str
 		req.Header.Set("Authorization", token)
 	}

+	req = AddAuthToken(ctx, req)
+
 	req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
 	req.Header.Set("Content-Type", "application/x-protobuf")
 	req.Header.Set("Accept", "application/x-protobuf")
@ -682,6 +704,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, in
 	req.Header.Set("Accept", "application/x-protobuf")
 	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request against the host.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -902,6 +925,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index
 	httpReq.Header.Set("Accept", "application/x-protobuf")
 	httpReq.Header.Set("X-Pilosa-Row", "roaring")
 	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	httpReq = AddAuthToken(ctx, httpReq)

 	// Execute request against the host.
 	resp, err := c.executeRequest(httpReq.WithContext(ctx))
@ -976,6 +1000,7 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *topology.Node,
 	}
 	req.Header.Set("Accept", "text/csv")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request against the host.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1018,6 +1043,7 @@ func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field,
 	}

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1107,6 +1133,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request against the host.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1145,6 +1172,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, inde

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1194,6 +1222,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi
 	req.Header.Set("Accept", "application/protobuf")
 	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
@ -1275,6 +1304,7 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, i
 	req.Header.Set("Accept", "application/x-protobuf")
 	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request against the host.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1329,6 +1359,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in
 	req.Header.Set("Accept", "application/x-protobuf")
 	req.Header.Set("X-Pilosa-Row", "roaring")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request against the host.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1360,6 +1391,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[s

 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request against the host.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1391,6 +1423,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]p

 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request against the host.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1432,6 +1465,7 @@ func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, i
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Send the request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1480,6 +1514,7 @@ func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, i
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Send the request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1529,6 +1564,7 @@ func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI,
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Send the request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1581,6 +1617,7 @@ func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI,
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Send the request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1625,6 +1662,7 @@ func (c *InternalClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI,
 	req.Header.Set("Content-Length", strconv.Itoa(len(like)))
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Send the request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -1664,6 +1702,7 @@ func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.T
 	}
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx))
 	if err != nil {
@ -1702,6 +1741,7 @@ func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeou
 	req.Header.Set("Content-Type", "application/json")
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
 	if err != nil {
@ -1736,6 +1776,7 @@ func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pil

 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
 	if err != nil {
@ -1772,6 +1813,7 @@ func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa
 	}
 	req.Header.Set("Accept", "application/json")
 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true))
 	if err != nil {
@ -1797,6 +1839,8 @@ type executeOpts struct {
 	// giveRawResponse instructs executeRequest not to process the
 	// respStatusCode and try to extract errors or whatever.
 	giveRawResponse bool
+	// forwardAuthHeader instructs executeRequest not to follow redirects
+	forwardAuthHeader bool
 }

 type executeRequestOption func(*executeOpts)
@ -1806,6 +1850,11 @@ func giveRawResponse(b bool) executeRequestOption {
 		eo.giveRawResponse = b
 	}
 }
+func forwardAuthHeader(b bool) executeRequestOption {
+	return func(eo *executeOpts) {
+		eo.forwardAuthHeader = b
+	}
+}

 type nopCloser struct {
 	*bytes.Reader
@ -1831,7 +1880,25 @@ func (c *InternalClient) executeRetryableRequest(req *retryablehttp.Request, opt
 		opt(eo)
 	}

-	resp, err := c.retryableClient.Do(req)
+	var resp *http.Response
+	var err error
+	if eo.forwardAuthHeader {
+		rc := retryablehttp.NewClient()
+		rc.HTTPClient = &http.Client{
+			CheckRedirect: func(req *http.Request, via []*http.Request) error {
+				if len(via) > 0 {
+					req.Header.Set("Authorization", "Bearer "+getToken(via[0]))
+				}
+				return nil
+			},
+		}
+		rc.CheckRetry = retryWith400Policy
+		rc.Logger = logger.NopLogger
+
+		resp, err = rc.Do(req)
+	} else {
+		resp, err = c.retryableClient.Do(req)
+	}

 	return c.handleResponse(req.Request, eo, resp, err)
 }
@ -1846,6 +1913,7 @@ func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp
 	if eo.giveRawResponse {
 		return resp, nil
 	}
+
 	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
 		defer resp.Body.Close()
 		buf, err := ioutil.ReadAll(resp.Body)
@ -2104,6 +2172,7 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context,
 	}

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -2171,6 +2240,11 @@ func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, ind
 	}
 	httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version)

+	token, ok := ctx.Value("token").(string)
+	if ok && token != "" {
+		httpReq.Header.Set("Authorization", token)
+	}
+
 	// Execute request against the host.
 	resp, err := c.executeRetryableRequest(httpReq.WithContext(ctx))
 	if err != nil {
@ -2196,6 +2270,7 @@ func (c *InternalClient) ShardReader(ctx context.Context, index string, shard ui

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/octet-stream")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -2218,6 +2293,7 @@ func (c *InternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser,

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/octet-stream")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -2227,6 +2303,30 @@ func (c *InternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser,
 	return resp.Body, nil
 }

+func (c *InternalClient) IDAllocDataWriter(ctx context.Context, f io.Reader, primary *topology.Node) error {
+	span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.IDAllocDataWriter")
+	defer span.Finish()
+
+	u := primary.URI.Path("/internal/idalloc/restore")
+
+	// Build request.
+	req, err := http.NewRequest("POST", u, f)
+	if err != nil {
+		return errors.Wrap(err, "creating request")
+	}
+
+	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
+	req.Header.Set("Accept", "application/octet-stream")
+	req = AddAuthToken(ctx, req)
+
+	// Execute request.
+	_, err = c.executeRequest(req.WithContext(ctx))
+	if err != nil {
+		return err
+	}
+	return err
+}
+
 // IndexTranslateDataReader returns a reader that provides a snapshot of
 // translation data for a partition in an index.
 func (c *InternalClient) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) {
@ -2244,9 +2344,10 @@ func (c *InternalClient) IndexTranslateDataReader(ctx context.Context, index str

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/octet-stream")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
-	resp, err := c.executeRequest(req.WithContext(ctx))
+	resp, err := c.executeRequest(req.WithContext(ctx), forwardAuthHeader(true))
 	if resp != nil && resp.StatusCode == http.StatusNotFound {
 		resp.Body.Close()
 		return nil, pilosa.ErrTranslateStoreNotFound
@ -2273,6 +2374,7 @@ func (c *InternalClient) FieldTranslateDataReader(ctx context.Context, index, fi

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/octet-stream")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -2303,6 +2405,7 @@ func (c *InternalClient) Status(ctx context.Context) (string, error) {

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
@ -2334,6 +2437,7 @@ func (c *InternalClient) PartitionNodes(ctx context.Context, partitionID int) ([

 	req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
 	req.Header.Set("Accept", "application/json")
+	req = AddAuthToken(ctx, req)

 	// Execute request.
 	resp, err := c.executeRequest(req.WithContext(ctx))
--- a/http/handler.go
+++ b/http/handler.go
@ -443,8 +443,8 @@ func newRouter(handler *Handler) http.Handler {

 	// Truly used internally by featurebase
 	router.HandleFunc("/internal/cluster/message", handler.chkInternal(handler.handlePostClusterMessage)).Methods("POST").Name("PostClusterMessage")
-	router.HandleFunc("/internal/translate/data", handler.chkInternal(handler.handleGetTranslateData)).Methods("GET").Name("GetTranslateData")
-	router.HandleFunc("/internal/translate/data", handler.chkInternal(handler.handlePostTranslateData)).Methods("POST").Name("PostTranslateData")
+	router.HandleFunc("/internal/translate/data", handler.chkAuthZ(handler.handleGetTranslateData, authz.Read)).Methods("GET").Name("GetTranslateData")
+	router.HandleFunc("/internal/translate/data", handler.chkAuthZ(handler.handlePostTranslateData, authz.Write)).Methods("POST").Name("PostTranslateData")

 	// other ones
 	router.HandleFunc("/internal/fragment/block/data", handler.chkAuthN(handler.handleGetFragmentBlockData)).Methods("GET").Name("GetFragmentBlockData")
@ -566,7 +566,8 @@ func (h *Handler) chkAuthN(handler http.HandlerFunc) http.HandlerFunc {
 				return
 			}
 		}
-		handler.ServeHTTP(w, r)
+		ctx := context.WithValue(r.Context(), "token", r.Header["Authorization"])
+		handler.ServeHTTP(w, r.WithContext(ctx))
 	}
 }

@ -590,6 +591,7 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http

 		// put the user's groups in the context
 		ctx := context.WithValue(r.Context(), contextKeyGroupMembership, uinfo.Groups)
+		ctx = context.WithValue(ctx, "token", "Bearer "+uinfo.Token)

 		// unlikely h.permissions will be nil, but we'll check to be safe
 		if h.permissions == nil {
--- a/internal/authclustertests/docker-compose.yml
+++ b/internal/authclustertests/docker-compose.yml
@ -0,0 +1,77 @@
+version: '2'
+services: 
+  pilosa1:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-clustertests
+    image: ptest
+    environment:
+      - PILOSA_NAME=pilosa1
+      - PILOSA_ETCD_DIR=/root/.etcd
+      - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
+      - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa1:10201
+      - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
+      - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301
+      - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
+      - PILOSA_CLUSTER_REPLICAS=3
+    networks:
+      - pilosanet
+    command:
+      - "/featurebase server --bind pilosa1:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf"
+  pilosa2:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-clustertests
+    image: ptest
+    environment:
+      - PILOSA_NAME=pilosa2
+      - PILOSA_ETCD_DIR=/root/.etcd
+      - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
+      - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa2:10201
+      - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
+      - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301
+      - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
+      - PILOSA_CLUSTER_REPLICAS=3
+    networks:
+      - pilosanet
+    command:
+      - "/featurebase server --bind pilosa2:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf"
+  pilosa3:
+    build:
+      context: ../..
+      dockerfile: Dockerfile-clustertests
+    image: ptest
+    environment:
+      - PILOSA_NAME=pilosa3
+      - PILOSA_ETCD_DIR=/root/.etcd
+      - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201
+      - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa3:10201
+      - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
+      - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301
+      - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
+      - PILOSA_CLUSTER_REPLICAS=3
+    networks:
+      - pilosanet
+    command:
+      - "/featurebase server --bind pilosa3:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf"
+  client1:
+    build:
+      context: .
+      dockerfile: ../clustertests/Dockerfile
+    depends_on:
+        - "pilosa1"
+        - "pilosa2"
+        - "pilosa3"
+    environment:
+      - ENABLE_PILOSA_CLUSTER_TESTS=1
+      - GO111MODULE=on
+      - PROJECT=authclustertests
+      - ENABLE_AUTH=1
+    networks:
+      - pilosanet
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    command:
+      - "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -count=1 github.com/molecula/featurebase/v3/internal/clustertests"
+networks: 
+  pilosanet:
--- a/internal/authclustertests/testdata/certs/README.md
+++ b/internal/authclustertests/testdata/certs/README.md
@ -0,0 +1,12 @@
+
+
+# these test certs were generated with the following commands
+
+certstrap --depot-path certs init --common-name pilosa-ca --expires "100 years"
+certstrap --depot-path certs request-cert --common-name localhost --domain localhost
+certstrap --depot-path certs sign "localhost" --CA pilosa-ca --expires "100 years"
+
+# certstrap version
+dev-25ea708a
+
+(built with go 1.13)
--- a/internal/authclustertests/testdata/certs/localhost.crt
+++ b/internal/authclustertests/testdata/certs/localhost.crt
@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEPjCCAiagAwIBAgIRAJ7rl74WPv8pLuhVRXt6fV0wDQYJKoZIhvcNAQELBQAw
+FDESMBAGA1UEAxMJcGlsb3NhLWNhMCAXDTIwMTAyMDE5MTMzNFoYDzIxMjAxMDIw
+MTkxMzE5WjAUMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA
+A4IBDwAwggEKAoIBAQDmi8FMWt23M0Cr2aCgEXUGQ0gv/4M7CXH/5GkSI866YwGV
+Bd1iZMBRiONQwvGDnqYZRrAQv6mFjfyBqxdkbh++74FC3JK7sLhks0vg5VwbHV7T
+5kj3bJqd+LKn5qPPOQXX9sgmv/NkggF/XXwF73noLPmgDQ78S+OP0ANmi1TQiU3a
+gE+qp+Qpl5KC7dH9aC9nvE9iGfEcGNr+rXj05liiXqe4ZtIKWjeke7Ej64C6qX97
+bNPzmLARtqbRsIkfAU8SJy3YuHfW8n1xr4B7ENm9jHQCh1wUv2YhaPpnio+/R2zp
+Lw4yCqilDX9ZZ4nG3cBFuziSf+BUXJ9ydbw1aX8HAgMBAAGjgYgwgYUwDgYDVR0P
+AQH/BAQDAgO4MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAdBgNVHQ4E
+FgQUJRQJpaR5bp4ZyUsMxgK+UJl6PSgwHwYDVR0jBBgwFoAU69lmXSa5BZeyYU/6
+XpWdtr59H1YwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IC
+AQChxsBZ/b14ukJXX48BxAyZcy5r7GrLcRGQ3guUTONFVDWPzzpd8mjHi0yJDhMW
+2zWtw3/H+c+zT7uRd+2sUxFdpAurNSFCdV++5Q/0aFvl+By5+MhVhtznEQDU0/lM
+zFxiEYe/N9Vi2N0S1KPxvYL/RfBU27u+O/50zhjueM1BTyHTTqL6E2DFeT2VPKIg
+zCDUtiTEDFZrD0XGITT/3CIoNCK8aC+Fq65OEoyEn6qR5qg1Kc4tfZmo6hWYiSlR
+XeP36cP9R8kEMte1BdE74GVqE9cTuVZERdgB0hv3EME7Byq7uIm/a+JXbsh2/OFm
+HcE0/HP+O0YK8YaVMGwI3pZYy2syWqPcakcvusETehr6P+Ihh2cOKRwqkCl6b87e
+uSLJNTUMKZgakW6Bjv6lgQaWqnKzTC/RgmQ+G3w0nKATX9+jYE2j3MzZhbtcml+2
+gp6u225yAJaYt/MQidwUMiKYeCgjaUNoL0fOJesGkokPk80ceISnqvbSRiZRTvK1
+bVenkhkBrHuvvgKVstzcuZI9oQ2snWhK1naVQiOtQNEFUCHwyU95zADOK0km88NB
+2het6yYaEUL9csHPEjPd3lFglerGQnil2Ly1slUC4jb7hfVRHjOFs8PVr9gQ45dW
+Jvsv4pawHKFE0ennoNvoDmzbiY1TY5ScTZquPGIsEBV+tQ==
+-----END CERTIFICATE-----
--- a/internal/authclustertests/testdata/certs/localhost.csr
+++ b/internal/authclustertests/testdata/certs/localhost.csr
@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE REQUEST-----
+MIICgDCCAWgCAQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0B
+AQEFAAOCAQ8AMIIBCgKCAQEA5ovBTFrdtzNAq9mgoBF1BkNIL/+DOwlx/+RpEiPO
+umMBlQXdYmTAUYjjUMLxg56mGUawEL+phY38gasXZG4fvu+BQtySu7C4ZLNL4OVc
+Gx1e0+ZI92yanfiyp+ajzzkF1/bIJr/zZIIBf118Be956Cz5oA0O/Evjj9ADZotU
+0IlN2oBPqqfkKZeSgu3R/WgvZ7xPYhnxHBja/q149OZYol6nuGbSClo3pHuxI+uA
+uql/e2zT85iwEbam0bCJHwFPEict2Lh31vJ9ca+AexDZvYx0AodcFL9mIWj6Z4qP
+v0ds6S8OMgqopQ1/WWeJxt3ARbs4kn/gVFyfcnW8NWl/BwIDAQABoCcwJQYJKoZI
+hvcNAQkOMRgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQAD
+ggEBABMi2/4j1/qzwWAYlEs2KW3z+apzzDLKgjE0kY6QvELh/8aBj0rMglb0HM2x
+4iSSoX1ZwZgDZ9fIJ3klG/UF7CUweMghb9yC2PP9Z8WuqaECQyM87KgSln8PND9E
+1OvD30rp9yr9KxEeckq+c1ebLi/qGrIY21VCwfxA0mv3sfi7Q5ONIckay/Xj+1Tz
+ovE/TkM/8wTE/SKbpQSCkP7K1NDXuAhMGjcN0x3d3f8nBcLcZOrRroiHy38Bv/9T
+Vd62IY6uqYw9sluBbMX72D/mmJiCKEw3+DhDJFhHCTCrAQM0QwLuwnG2lQFYoENc
+ZAkwDIi+3DXHEEyloNSYGtXMEiA=
+-----END CERTIFICATE REQUEST-----
--- a/internal/authclustertests/testdata/certs/localhost.key
+++ b/internal/authclustertests/testdata/certs/localhost.key
@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEA5ovBTFrdtzNAq9mgoBF1BkNIL/+DOwlx/+RpEiPOumMBlQXd
+YmTAUYjjUMLxg56mGUawEL+phY38gasXZG4fvu+BQtySu7C4ZLNL4OVcGx1e0+ZI
+92yanfiyp+ajzzkF1/bIJr/zZIIBf118Be956Cz5oA0O/Evjj9ADZotU0IlN2oBP
+qqfkKZeSgu3R/WgvZ7xPYhnxHBja/q149OZYol6nuGbSClo3pHuxI+uAuql/e2zT
+85iwEbam0bCJHwFPEict2Lh31vJ9ca+AexDZvYx0AodcFL9mIWj6Z4qPv0ds6S8O
+MgqopQ1/WWeJxt3ARbs4kn/gVFyfcnW8NWl/BwIDAQABAoIBAFX+GPqfBgY4cs3m
+3ff2qvzMCdgFaXCS5Fe7XcmrW4fAOC3awynZRLbk5U0Reb5LZc8Vw8RriRLM1DuV
+kqMeRG8WrNNArOafUxgUnJ/lTUa73MwTIHJRqxZzVkg0SjOYJGranOt/O4zoxSA5
+wXIBUipc5Dtjw4wtzlKtFyefnuItL2MCdwOHUdZfnhr9Oykp1fuNqBqkkeryj3XV
+ukHQvqU5zkMSayprNglziqTHUzU33iyZeDng+CJQeYTEc7Gn+zja2SFFBlPHqXXo
+/OzAr94zI3vOnj3yRM3+sKMJVPV+RoJEGpsvPVuVn38d1VnIMEx8Gy/wif6tmM9c
+7Q44hKECgYEA/JMFwkPGbry80ktDI065k5FIYn1EDRyUaQqyskmkBRcNW3qOShqj
+o/zWQfCgxP587IEdKBBwqCpdqfghi3EW+JqfVlbGY6t1chAurYF/47CTIgKO5qRM
+GdCY2OdiAeo5nba/KiLQfSuY08MCNDrQabLRJXIng8qVWpRwQzsv5rECgYEA6aw/
+HugeQhTxk2uV91jJaAQIaxrt6JxuoG0CGGlbDrrTl2dnbPYA0muHMFdT/bzKjCpv
+n/ScqbCyHuy+lWSnOzgedRNQCB46+0H58LAjITAj9QaT3raZqVReVIaD+pnx27dp
+Cw5Ws6ENa9AQey3DO+dkRWot2AcLSw6TGR8HnzcCgYEA/ArfCU/G2cSgDJ6sPbSW
+vaqR+C6W1Rq7AuN5FS8lbSrm2m2/RjW1LLTnPmAYntxx3zSs2sklErtMQovpNZRB
+3w21iVwIl3eHOK7rVZtP+u++s4aoAYLcqjod/P1RMSYCHt85fpvFP9Ncq50DOwmh
+5ohZ6ysyQXLMfdp4+K48i9ECgYEApOFALKu2ZgRnLRFV4REKFFX8Jq76vg5bVOF2
+AAmfEbasBIIXDWBL1i2/V1HXVwv2k46B8wjj3ixqkr2UAM/j3DpN62g0KXZDQfUc
+ykNOlmVkickZX6XSqRN5+ARubc5gRRuWiBGXBeqXEMLgTjpNLyCntP8l1++ofU6M
+ZsZpV2MCgYA3nfNXAR5O4B/dm/2HmDQrXy0qia7Hwi/95pgL2FJaEmjBCPI1j12o
+M5YCbhpr1pwsNKPV9AUlUz+OCwS8Vt+V0gQf9/XvNOsifU+mbMYVpuNGwmcKafnv
+qECSeidrmhWJSR/SSNBcE94im/8ObVU110WJMkjC9otjDl9Aua/3LQ==
+-----END RSA PRIVATE KEY-----
--- a/internal/authclustertests/testdata/certs/pilosa-ca.crl
+++ b/internal/authclustertests/testdata/certs/pilosa-ca.crl
@ -0,0 +1,16 @@
+-----BEGIN X509 CRL-----
+MIIChTBvAgEBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMTCXBpbG9zYS1jYRcN
+MjAxMDIwMTkxMzIyWhgPMjEyMDEwMjAxOTEzMjBaMACgIzAhMB8GA1UdIwQYMBaA
+FOvZZl0muQWXsmFP+l6Vnba+fR9WMA0GCSqGSIb3DQEBCwUAA4ICAQBja+EDQAp+
+KeD7UhWMMrTd9j03GgQ2E2Z7+Ba0qJ5+kS7/t+Yja2o5dQJkrC3GwEMOQb6DRRUE
+nUE4xlr5Rryoq0dZk+Lp1f4cHrnP8l1xylUL44gsnY4v8zMR8L8X98vj7kKCqB8w
+DFX7qkMlE5Ie2Hha7uuOJ85FnxIbMcRxFQH2m2zDfWG8/Lmxezvv9Hn45V/kwIQy
+MmBh6cNuhzEneyNpM9yMRe/29QgVitF/2q6d+FzK8w8hkUFeYlyM+cP7F4Ml7160
+UidSQM04zvBtJ8frZAvrDaPBBZhrTXcyw6+Qnp/aaW1ZsEIdHEcbYGNdgtazleoG
+VH35cDP90KfiRbq69PQ9Zqn3cI//MX3sHrglA9wsEhHc9P7dowHaOFyxPouZPEmQ
+/Jqg5oyJzujRwhf0v3SdJvhuDEzla2N+QyYRk0kRHtdv+glz7T7CnTYCk+DTv+oh
+QABUrCbjfBoE5M2Qep9ZkIbl2gaDCpvbZSF4zFLKQc2aIOBpVn3HgTGBvdFD3FJY
+Txl2F4Y3rS1T/WMAH86cZIc9h5HlMdFtAFnHAlHtB3wGw3FD/GcvGcvz2D4GaxKq
+erzrnOxjYOA4M0haGzWF6dC7aPA8y35eZuqNXvbenTtc7A11bWTJfG1I7ctvLyPE
+MpCNMHfymh/XtYZiZhvu6ueu3OeKScN+tA==
+-----END X509 CRL-----
--- a/internal/authclustertests/testdata/certs/pilosa-ca.crt
+++ b/internal/authclustertests/testdata/certs/pilosa-ca.crt
@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE6jCCAtKgAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwlwaWxv
+c2EtY2EwIBcNMjAxMDIwMTkxMzIyWhgPMjEyMDEwMjAxOTEzMjBaMBQxEjAQBgNV
+BAMTCXBpbG9zYS1jYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALT/
+uNmbnfXWNX+FsL0Waqw/5deti5F4cSjMrGRQpXxalTcooqNk/lkeqXkvi9ooFROZ
+/HyQR9GM9dSD/aj6gD3FnGA4ueB24Xr6bWsRpDRh6+3UGLB3YCNNdGLSfX3LPMYh
+RJutFmsg+r6SrSytbLbffu+0a/4fxtajZNwQJjDjd8qflXQZYlzp2LHk1A/jqqdI
+fBtqkNg925TGKiavvUqKtdI/eFzRoiQ7NLBUJmszzveUXvOUMsMnW2/myLBe3Oqk
+Vsy85lya0ADln20C3Lb0+ZA4KoGX3EWdtBXEuWqMoyvCJoJ4I3bH2LlfOUjRt8UE
+pPk6sPMROJ+75mlgvgnSlYsN8PaZdvdm2VGVWRWUyEfyW/qa2fv8d2XBWqibl0YF
+tqay9CX1aWgC9q12yx3vj7Yh+ZNbeZFLc7IL8zyNMwIjIOIIyGBY70KewfgVktzq
+fAMz6h1sr9Kxozil97Cu3ma4B6UiL3rUbYMO/rNhVxcIuUoJpgIEVuRt+uXEG74y
+XftauZ67qILFQzfpoacncvDEx5nJ3itLgbbyt1n1iWdGuEiMSLFT+x+nNUgVpQgA
+sWRYxHdisM4xzRVN6pAaToMs1p8Ju7l9xU3z7RSogTyVk9gMIIV4t9TDYP5fI10Y
+GFi7B6q0t3pIGXgKySHjCSl0EKYkDQDFl5tWeaiXAgMBAAGjRTBDMA4GA1UdDwEB
+/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTr2WZdJrkFl7Jh
+T/pelZ22vn0fVjANBgkqhkiG9w0BAQsFAAOCAgEAnlBFrWhB+WesCc3lhK980rA6
+roNFYMZdaXvg4zaEGergkRvPab5yXoof1AAeznJm45GQfXn8HbQlrZmAqWg3fNld
+/TX+jNvosM8K8K+PzesDGHsm/eQnbrb0qzMDsQgFY+nnD+x/ZQtjmKZtcNr/0ZlM
+EJeXWU5cGy70GMbNztspMHsOLa3ZDLsBOJYOwSFxDlLDFrjZoRoPCWw8jRL+Tb4t
+JjZcGZDD4a5+DqcojanIdNU1yI4teP6aV1LQTVNn4pwOap+tD0De/WzOPmXTQq5M
+9ssxL7xSqVShQQMC8LVSWSRxtT6kLq0Av6i7wio0DZGnH3ynERTUs13DRZkwbVsE
+OaQLmiQnsHRTIpdts/fswZ2FRPvdhhXxBjiGQZGEGXXznxHTNJ6nQioh95Ft5hNA
+82i8Z74miaFIT/33/sZ5SuwUzphCgqCY2x7NUS8J313O9lsar0bweJTvQaZg/69E
+PmmwUcDebh+pgKP01z4BqTzhtchmFUKzT+oOC8tmTeSlhsBTzO4xMw6OqhpXKL+c
+k9f2CGUZYtEZHDRmP+C++FEi+B/tV2Oq3on+QPiaIIcRsOftthGUvJ8htUl3w+hq
+B5TnL8CeLjXGKKRp+UiakrB4E7y2aIbrtIRnJ/Llg2XMND/0xbldsNsyDNXCIoDH
+sz8HqwF3CUbv5XD4ioY=
+-----END CERTIFICATE-----
--- a/internal/authclustertests/testdata/certs/pilosa-ca.key
+++ b/internal/authclustertests/testdata/certs/pilosa-ca.key
@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJKAIBAAKCAgEAtP+42Zud9dY1f4WwvRZqrD/l162LkXhxKMysZFClfFqVNyii
+o2T+WR6peS+L2igVE5n8fJBH0Yz11IP9qPqAPcWcYDi54HbhevptaxGkNGHr7dQY
+sHdgI010YtJ9fcs8xiFEm60WayD6vpKtLK1stt9+77Rr/h/G1qNk3BAmMON3yp+V
+dBliXOnYseTUD+Oqp0h8G2qQ2D3blMYqJq+9Soq10j94XNGiJDs0sFQmazPO95Re
+85Qywydbb+bIsF7c6qRWzLzmXJrQAOWfbQLctvT5kDgqgZfcRZ20FcS5aoyjK8Im
+gngjdsfYuV85SNG3xQSk+Tqw8xE4n7vmaWC+CdKViw3w9pl292bZUZVZFZTIR/Jb
+prZ+/x3ZcFaqJuXRgW2prL0JfVpaAL2rXbLHe+PtiH5k1t5kUtzsgvzPI0zAiMg
+4gjIYFjvQp7B+BWS3Op8AzPqHWyv0rGjOKX3sK7eZrgHpSIvetRtgw7+s2FXFwi5
+SgmmAgRW5G365cQbvjJd+1q5nruogsVDN+mhpydy8MTHmcneK0uBtvK3WfWJZ0a4
+SIxIsVP7H6c1SBWlCACxZFjEd2KwzjHNFU3qkBpOgyzWnwm7uX3FTfPtFKiBPJWT
+2AwghXi31MNg/l8jXRgYWLsHqrS3ekgZeArJIeMJKXQQpiQNAMWXm1Z5qJcCAwEA
+AQKCAgAWmjiDNCOtp2pW2mMPudToXbJeFJXxPJEk/yon/MotlUI8+R4WOW5pwqJ3
+N7DHNWosYHZfN8VALdIlD7aFe4K4NA0rFupfVXki2lL/o9xVjkTgFjRfFQk0X1/B
+V3fEVbTpKQ5gQmUiS6QEWFy3z5Bb5dz8IhO6UE2MUCswL/QU9tLmwrbvIJxf7fPZ
+gzHYKh4NdcfJxK0B0/evxG9PFXMV8+xwrOxi6urMi3gw7NE/YeDemfChikgshqWs
+e61kGPSNeKg+OPirZ8nB0urtugXF8yGXGOx18njXWLI8Zayh2Z4mwL/+WvJSyvIN
+dA67QTUprULMvL+MGwJvMA+96Q7SBRVKR9HHNaP9pFsZup3QX9mqDQ9miB6+rzn7
+f5RiSLVgq+HUPMPfqgXCkQZBcY28TcM1BZhS4uJJTkbvVTkrlHKJs6Q/LBqJFvq0
+3+2M1xQb4HdRTRlwZ/YsxdqXGIoA3Xx3nZbb6LlPp/MT93xxsLJNNP47n445Cw8i
+lz7hJJDwo+TyXmRRWKlFXO8TEhqKhK9ZEXmkXBxSeCQV0oTYS5kU9XdZ5iu5/CQQ
+Lv+uFQfHTPWm/Lp5RC8JEEJwK5bwRAs5d9oWg5EbWJp2ol36g3YO1np5RFsQn3jL
+qJPz35X3Bp9zQeZcAZpt1fWFdyX9f7V2LLCY8gUyIlfCEXBWwQKCAQEA8Uoes4ph
+15tM1LswmOLqOq9iWuvcKYSNcz8my8nlP5zkUdGKvGcLblm5Mg4wKoPFiyjKhX7I
+S8DUN8x7E5aiNBiZ0PGku4CKjubQgdFG/rYRfnrUEaA81Uw3a69eYioKbxhhxDzb
+Lqd0/tGNHQZBQEyofOChwqTWpAdIh79F1oXejcPzRDr+oXxJVnvJDdYCEN/0TYkz
+qeJdEtnVf1x2oNjuPIRuZNldpSDmiUce4QXG/qKgJcQNZ+paDDFK/G/rXHcIvV5C
+du9yxxfppY7fRRMm+LFqDhKEWveG4OUhUgut71J0EREO65oMbf6UcZ2FS4XDTbFD
+RSO4d8bKgf2eKwKCAQEAwAigvp8qq/yYitZeXhI6cI4ztSvwqblREhUgYWyrbUFo
+a38Bey1fKQzJYUYA7raFU6alRHoHBQywjANhIaLlfvLtQfuZ4Z9CGP3Qn33uQR/E
+ha4MNjjwUB0jx9lsDze1h61V95fxQLGNLVwGoaES4BpDRqvYJKnQv2X0SUigEg5U
+GwryNlEW0AS/Xp/k7+PGJQernHIEWYS70FleHbAiINh+lzSfbJObgd6XnQ8IxtTr
+xthXBKkkNBJdJX+/3qUQgOxTjSNUY4N9Np7myFfMvcAXuR7/K7bDegwHxffYb6Gc
+v3fCFoTQFn1KTh0IvRjyv3WzqInAYVjC8CpD562VRQKCAQBirI4LnE7Q7lioMnj4
+POvO3gRZ7FSXwfZap/vEoScYMaAJeajDzVwWX6jluHmoGUVC2IahuyxMFmpy+zNl
+2lcw+NKGaRuV9kYzlF62iBABgBF9aNuq7Z2TGN0dM5VkjY7AyfbJWp3D4YVt4+JS
+eUlb8z1//BkK0YBZigT2RplX1l0iGn00bO/OuFYBgRPCjb9AiWWOA8rV8ZVgbSbr
+M7PrqWsb4oiGw4GRUvgUMbqGCWfMoFLfvuJAmc0DaXEh9N8KbD9tuctyeg+1LalG
+JDxYMjHgyCT35kisLsfA1tMei1oxIcYHaLNyVAg7Pz4TjHiDXwt0jUZWUvpQOUJ9
+kGsLAoIBAQCOqFoyAjBDIB16VpI4NDZx01IabxAUJfVSB5vMhFw9h++4m9tP1H7z
+Eeqwdr7Ol40ofY4c9sIsQCcPfJs1z7vJuVIESJMih5sk0bmgIn9SpfTqkkfEKDxu
+Z5djKeQa0fnrVxucGaZBtyT343uRqwVIsnn0EEk7w2OuLGFz553yi+5zQIh7TXYz
+BrPb6dC7XWyfqbkVOaZ9khusRhei2mwgFnTEg3VDxcwqiF/9b2PHwfl9+M18SuL4
+RAQqjWLOVbWS8P2Ixgw0+UOVxioP/xm8hO2auqo5oUZKbpF/wgVpuJenraHj9LpZ
+Wq5OpUcOo3ACR8A1nk/qgXQf0mYrwEo5AoIBAHEqA2eJVZnPiAs6U7QPAavnLxt/
+v0GLzsBBixSV8ErMToN1wfYtBb1t5fgF0Fuy85dREp1CsGJMgrnPX5bCnBmDaLl2
+Z1lUaSDcFCu+yXo+Kuy7JvSKZ4++q4ggrHvK8y8FdKH4H+56vTdXe2i9RY/v48g4
+kKyNiYtVXxrd/h47WbHF5eApheblH9hH6zC5tB/rW7Hh0nmnDcfmMW4BggbyBinH
+MF3jO0YaspZOtRc2xSj8E3sGtN+f/KrBbKBb4J0j7VzuFmZC1u5grl/hx0cYE2ek
+HGifmIjkKv5R4xPELoAJZyFOpN1PfS3Y+SOn0mF+RJRoGqMGcQWA3I77b5M=
+-----END RSA PRIVATE KEY-----
--- a/internal/authclustertests/testdata/featurebase.conf
+++ b/internal/authclustertests/testdata/featurebase.conf
@ -0,0 +1,383 @@
+# FEATUREBASE HOST CONFIGURATION
+#
+# Uncomment when/where appropriate
+
+# ==============================================================================
+# Use advertise to specify the address advertised by the server to other nodes 
+# in the cluster and to clients via /status endpoint. Host defaults to IP 
+# address represented by bind parameter with network port.
+#
+# advertise = <IP Address>:10101
+# advertise-grpc = <IP Address>:20101
+
+
+
+# "long-query-time" represents duration of time that will trigger log and stat 
+# message for queries longer than X time. Ex. "1m30s" 1 minute 30 seconds
+#
+# long-query-time = "10s"
+
+
+
+# Unique name for node in cluster. This is just a human-readable label for 
+# convenience and not used by any underlying logic.
+
+# name = "pilosa1"
+
+
+
+# # Host:Port where Featurebase server listens for HTTP requests.
+# # Default is localhost:10101
+# #
+# bind = "pilosa1:10101"
+
+
+
+# # The address and port featurebase will listen to for all GRPC connections
+# # Ex. python-molecula, grafana for queries, etc.
+# #
+# bind-grpc = "localhost:20101"
+
+
+
+# Directory to store Featurebase data files
+# data-dir = "/var/lib/molecula"
+
+
+# ==============================================================================
+# CORS (Cross-Origin Resource Sharing) Allowed Origins
+# List of allowed origin URIs for CORS
+#
+# [handler]
+# allowed-origins = ["https://myapp.com", "https://myapp.org"]
+
+
+
+# Path to the log file
+# log-path = "/var/log/molecula/featurebase.log"
+
+
+
+# Verbose - Enable verbose logging. Valid options are true or false.
+# Set to true only when debugging as directed by Molecula engineers.
+#
+# verbose = true
+
+
+
+# Soft limit on max number of files featurebase will keep open simultaneously.
+# When past this limit, featurebase will only keep files open for as long as is
+# needed to write updates.
+#
+# max-file-count = 900000
+
+
+
+# Maximum number of active memory maps featurebase will use for fragment files.
+# Actual total usage may be slightly higher.
+# Best practice is to set this to ~10% lower than your system's max map count.
+# See sysctl vm.max_map_count in Linux.
+#
+# max-map-count = 900000
+
+
+
+# Max Writes Per Request - Max number of mutating commands allowed per request. 
+# This includes Set, Clear, ClearRow, and Store
+#
+# max-writes-per-request = 5000
+
+
+
+# The following option sets the maximum number of queries that are maintained
+# for the /query-history endpoint.
+# This parameter is per-node, and the result combines the history from all nodes.
+#
+# query-history-length = 100
+
+
+
+# External database to connect to for `ExternalLookup` queries.
+# lookup-db-dsn = "postgres://localhost:5432/db"
+
+
+
+# ==============================================================================
+# For cluster stanza, "name" represents name for cluster. Must be same on all 
+# nodes in cluster. "replicas" represents number of hosts each piece of data 
+# should be stored on. Must be greater than or equal to 1 & less than or equal 
+# to number of nodes in cluster.
+# [cluster]
+# name = "cluster1"
+# replicas = 1
+
+
+
+# ==============================================================================
+# [etcd]
+# etcd is the tool Featurebase uses for node-to-node, intra-cluster 
+# communication.  etcd is embedded in the featurebase cluster rather than 
+# running as a separate instance.
+# It's important to configure this correctly for your network and nodes, and 
+# that it is consistent across all nodes.
+#
+# The easiest setup can be used when all nodes can reach all other nodes via a 
+# local subnet:
+#     listen-peer-address = advertise-peer-address
+#                         = (what's in the initial-cluster-list)
+#                         = the nodes ip address (which can be reached by every 
+#                           other node
+#     (localhost:10401 would not work for this, as each node can't reach that)
+#
+# If each node is separated by a proxy, or must be reached via url / dns, you 
+# will need to use a more complicated setup:
+#    listen-peer-address    = the nodes local ip address
+#                             (specific ip, localhost, or 0.0.0.0 for all 
+#                             local ip's)
+#    advertise-peer-address = the nodes ip address, reachable by all other nodes
+#                             (This address should also be included in 
+#                             initital-cluster-list)
+# in this case, you specify a different url/ip for listen-peer and 
+# advertise-peer. E.g. you specify 0.0.0.0 for listen, or (like in their case) 
+# you use a url for advertise. In each of these cases, you should set listen 
+# to the local ip, and you set advertise = to how each other node connects to 
+# this node, and you also use this same address in the initial cluster.
+# The key here is that initial-cluster has to include the same node name and 
+# advertise-peer address as the node it's on (edited)
+
+
+
+# for additional assistance, and for help with config issues, 
+# see https://etcd.io/docs/v3.5/faq/ cluster-url - URL of existing cluster 
+# that a new node should join when adding nodes to cluster.
+#
+# cluster-url = "http://localhost:10401"
+
+
+
+# Address and port to bind to for client communication
+# listen-client-address = "http://localhost:10401"
+
+
+
+# Address and port to bind to for peer communication
+# listen-peer-address = "http://localhost:10301"
+ 
+
+
+# Comma-separated list of node=address pairs that makes up initial cluster when 
+# first started. In each pair, "node" value (left side of = ) should match 
+# name of node specified by "name" configuration parameter
+#
+# initial-cluster = "featurebase1=http://localhost:10301"
+ 
+ 
+ 
+# ==============================================================================
+# Profile Block Rate - Block Rate is passed directly to Go's 
+# runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 
+# per rate nanoseconds. A value of "1" samples every event, and 0 disables 
+# profiling.
+#
+# block-rate = 10000000
+
+# Profile Mutex Fraction - Mutex Fraction is passed directly to Go's 
+# runtime.SetMutexProfileFraction. 1/ fraction of events will be sampled.
+#
+# mutex-fraction = 100
+
+
+
+# ==============================================================================
+# PostgreSQL Section
+# [postgres]
+#
+# Endpoint Bind - Address to bind a PostgreSQL wire protocol endpoint.
+# No PostgreSQL endpoint will be exposed unless a bind address is specified.
+# Requires Molecula v3.0 or newer.
+#
+# bind = "localhost:55432"
+
+
+
+# The PostgreSQL endpoint has support for a connection limit.
+# This is generally not necessary, so it is disabled by default.
+#
+# connection-limit = 10000
+
+
+
+# PostgreSQL Max Startup Packet Size - By default, the postgres endpoint
+# uses an 8 MiB limit on incoming PostgreSQL startup packets. This should
+# typically be sufficient, but may be exceeded if a client sends an unusually 
+# large amount of configuration data. Oversized startup packets are typically 
+# caused by connecting with a different protocol, e.g. HTTP.
+#
+# max-startup-size = 10000000
+
+
+
+# PostgreSQL Timeouts
+# In order to detect stalled clients, the PostgreSQL endpoint has connection 
+# read and write timeouts. There is also a startup timeout, which is used for 
+# connection setup. The read timeout does not impact idle connections. Idle 
+# connections will only be closed by the server if TCP keepalive reports a 
+# break in the connection. TCP keepalives use the default configuration
+# provided by the host.
+# Caution: Due to a limitation of the PostgreSQL wire protocol,
+# raising the write timeout may delay the shutdown of a featurebase node.
+#
+# startup-timeout = "20s"
+# read-timeout = "20s"
+# write-timeout - "20s"
+
+
+
+# Postgres Endpoint TLS - TLS configuration for the PostgreSQL endpoint is
+# structured the same as the TLS configuration for Featurebase's other endpoints,
+# but placed under [postgres.tls]. If TLS is configured on the postgres endpoint,
+# Featurebase will reject unsecured connections.
+# [postgres.tls]
+#  certificate = "/Users/souhailanoor/tls/out/auth.mybusiness.com.crt"
+#  key = "/Users/souhailanoor/tls/out/auth.mybusiness.com.key"
+#  ca-certificate = "/Users/souhailanoor/tls/out/auth.mybusiness.com.crt"
+#  enable-client-verification = true
+
+
+
+# ==============================================================================
+# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is
+# calculated periodically in the background and accessed by the UI/usage 
+# endpoint. Since this disk scan can take a long and unpredictable amount of 
+# time, its timing behavior is specified in a relative, rather than absolute 
+# sense. That is, the duty cycle sets the percentage of time that is spent 
+# recalculating this cache. This setting affects the results received from 
+# the "/ui/usage" http endpoint, as well as all data file and memory usage 
+# values and graphs on the webui "tables" page
+
+# Special considerations:
+#      * If disk usage can be calculated quickly (less than 5 seconds), fresh
+#        results will be calculated when accessed
+#      * When disk usage takes longer to calculate, there is a minimum of one 
+#        hour wait between cache recalculations
+# Setting this value to 0 will completely disable the calculation of disk usage
+#
+# usage-duty-cycle = 20
+
+
+
+# ==============================================================================
+# Use [metric] stanza to define attributes for monitoring.
+# [metric]
+# Specify which service to use for collecting metrics. Valid options are:
+# "statsd", "expvar", "prometheus", "none"
+#
+# service = "prometheus"
+
+
+
+# Remote host to send statsd metrics to.
+# host = "localhost:8125"
+
+
+
+# The interval to send statsd metrics.
+# poll-interval = "10s"
+
+
+
+# Debugging flag to enable to send diagnostic information to Featurebase 
+# developers.
+#
+# diagnostics = false
+
+
+
+# ==============================================================================
+# TLS Certificate Section - Path to TLC certificate used for service HTTPS. 
+# Suffix should contain .crt or .pem
+
+[tls]
+ certificate = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/certs/localhost.crt"
+ key = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/certs/localhost.key"
+
+# ==============================================================================
+# Tracing Section
+# [tracing]
+#
+# Jaeger sampler type. Valid options are: "const, "probabilistic", "ratelimiting", 
+# or "remote". Set to 'off' to disable tracing completely.
+#
+# sampler-type = "remote"
+
+
+# Jaeger sampler parameter (number)
+# sampler-param = 0.001
+
+
+
+# Tracing Agent Host:Port
+# agent-host-port = "localhost:6831"
+
+
+
+# ==============================================================================
+# Configuration for the RBF storage format.
+# [rbf]
+# Maximum size for each RBF database file.
+# Allocates virtual memory but does not preallocate physical disk space. 
+# If you get into the range where you have 16000 shards on a single node 
+# (across all indexes), you will need to lower this in order to not run out of 
+# virtual address space.
+#
+# max-db-size = 4294967296
+
+
+
+# Maximum size for each RBF WAL file.
+# Allocates virtual memory but does not preallocate physical disk space. 
+# This is the same as max-db-size, but for the write-ahead log. If you set it 
+# smaller, set max-wal-checkpoint-size to 1/2 of this (we will likely 
+# condense these options in the future).
+#
+# max-wal-size = 4294967296
+
+
+
+# Minimum WAL size before WAL pages can be copied to the main database file.
+# min-wal-checkpoint-size = 1048576
+
+
+
+# Maximum WAL size before transactions are halted to copy WAL pages to the 
+# main database file.
+#
+# max-wal-checkpoint-size = 2147483648
+  
+
+# ==============================================================================
+# [storage]
+# Sync all changes to the file system.
+# Should not be changed in production systems unless you know what you are 
+# doing -  Should always be on unless testing or possibly while performing a 
+# bulk import and you are not worried about data loss
+#
+# fsync = true
+  
+  
+# ==============================================================================
+# Enable/Disable AuthN/AuthZ for featurebase 
+# Can choose identity provider, pass authorize and user-info endpoints, and client id
+[auth]
+ enable = true
+ client-id = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
+ client-secret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+ authorize-url="https://login.microsoftonline.com/organizations/oauth2/v2.0/authorize"
+ token-url="https://login.microsoftonline.com/organizations/oauth2/v2.0/token" 
+ group-endpoint-url = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true"
+ logout-url = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
+ scopes = ["https://graph.microsoft.com/.default", "offline_access"]
+ secret-key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+ permissions = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/permissions.yaml"
+ query-log-path = "query-log-test.log"
+ redirect-base-url = "https://localhost:10101"
--- a/internal/authclustertests/testdata/permissions.yaml
+++ b/internal/authclustertests/testdata/permissions.yaml
@ -0,0 +1,4 @@
+user-groups:
+  "group-id-test":
+    "test": "write"
+admin: "group-id-test"
--- a/internal/clustertests/cluster_test.go
+++ b/internal/clustertests/cluster_test.go
@ -11,9 +11,12 @@ import (
 	"testing"
 	"time"

+	"github.com/golang-jwt/jwt"
 	pilosa "github.com/molecula/featurebase/v3"
+	"github.com/molecula/featurebase/v3/authn"
 	"github.com/molecula/featurebase/v3/disco"
 	picli "github.com/molecula/featurebase/v3/http"
+	"github.com/molecula/featurebase/v3/logger"
 )

 // container turns a docker-compose service name into a container name
@ -24,16 +27,66 @@ import (
 // as well, but I think it is true in recent versions.
 func container(svc string) string {
 	project := "clustertests"
+	if os.Getenv("ENABLE_AUTH") == "1" {
+		project = "authclustertests"
+	}
+
 	if p := os.Getenv("PROJECT"); p != "" {
 		project = p
 	}
 	return project + "_" + svc + "_1"
 }

+func GetAuthToken(t *testing.T) string {
+	t.Helper()
+	var (
+		ClientID         = "e9088663-eb08-41d7-8f65-efb5f54bbb71"
+		ClientSecret     = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+		AuthorizeURL     = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize"
+		TokenURL         = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token"
+		GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true"
+		LogoutURL        = "https://login.microsoftonline.com/common/oauth2/v2.0/logout"
+		Scopes           = []string{"https://graph.microsoft.com/.default", "offline_access"}
+		Key              = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF"
+	)
+
+	a, err := authn.NewAuth(
+		logger.NewStandardLogger(os.Stdout),
+		"http://localhost:10101/",
+		Scopes,
+		AuthorizeURL,
+		TokenURL,
+		GroupEndpointURL,
+		LogoutURL,
+		ClientID,
+		ClientSecret,
+		Key,
+	)
+
+	// make a valid token
+	tkn := jwt.New(jwt.SigningMethodHS256)
+	claims := tkn.Claims.(jwt.MapClaims)
+	groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "group-id-test", GroupName: "group-name-test"}})
+	claims["molecula-idp-groups"] = groupString
+	claims["oid"] = "42"
+	claims["name"] = "valid"
+	token, err := tkn.SignedString([]byte(a.SecretKey()))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	return token
+}
 func TestClusterStuff(t *testing.T) {
 	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
 		t.Skip("pilosa cluster tests are not enabled")
 	}
+
+	auth := false
+	if os.Getenv("ENABLE_AUTH") == "1" {
+		auth = true
+	}
+
 	cli1, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil))
 	if err != nil {
 		t.Fatalf("getting client: %v", err)
@ -46,11 +99,18 @@ func TestClusterStuff(t *testing.T) {
 	if err != nil {
 		t.Fatalf("getting client: %v", err)
 	}
+	ctx := context.Background()
+	token := ""
+	// generate auth token and add to context
+	if auth {
+		token = GetAuthToken(t)
+		ctx = context.WithValue(ctx, "token", "Bearer "+token)
+	}

-	if err := cli1.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{}); err != nil {
+	if err := cli1.CreateIndex(ctx, "testidx", pilosa.IndexOptions{}); err != nil {
 		t.Fatalf("creating index: %v", err)
 	}
-	if err := cli1.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100}); err != nil {
+	if err := cli1.CreateFieldWithOptions(ctx, "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100}); err != nil {
 		t.Fatalf("creating field: %v", err)
 	}

@ -66,7 +126,7 @@ func TestClusterStuff(t *testing.T) {
 		req.ColumnIDs[i%10] = uint64((i/10)*pilosa.ShardWidth + i%10)
 		req.Shard = uint64(i / 10)
 		if i%10 == 9 {
-			err = cli1.Import(context.Background(), nil, req, &pilosa.ImportOptions{})
+			err = cli1.Import(ctx, nil, req, &pilosa.ImportOptions{})
 			if err != nil {
 				t.Fatalf("importing: %v", err)
 			}
@ -75,7 +135,7 @@ func TestClusterStuff(t *testing.T) {

 	// Check query results from each node.
 	for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} {
-		r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
+		r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
 		if err != nil {
 			t.Fatalf("count querying pilosa%d: %v", i, err)
 		}
@ -97,12 +157,12 @@ func TestClusterStuff(t *testing.T) {
 		}

 		t.Log("done with pause, waiting for stability")
-		waitForStatus(t, cli1.Status, string(disco.ClusterStateNormal), 30, time.Second)
+		waitForStatus(t, cli1.Status, string(disco.ClusterStateNormal), 30, time.Second, ctx)
 		t.Log("done waiting for stability")

 		// Check query results from each node.
 		for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} {
-			r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
+			r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"})
 			if err != nil {
 				t.Fatalf("count querying pilosa%d: %v", i, err)
 			}
@ -119,9 +179,17 @@ func TestClusterStuff(t *testing.T) {
 		}
 		var backupCmd *exec.Cmd
 		tmpdir := t.TempDir()
-		if backupCmd, err = startCmd(
-			"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest")); err != nil {
-			t.Fatalf("sending backup command: %v", err)
+
+		if auth {
+			if backupCmd, err = startCmd(
+				"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest"), "--auth-token", token); err != nil {
+				t.Fatalf("sending backup command: %v", err)
+			}
+		} else {
+			if backupCmd, err = startCmd(
+				"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest")); err != nil {
+				t.Fatalf("sending backup command: %v", err)
+			}
 		}
 		time.Sleep(time.Second * 5)
 		if err = sendCmd("docker", "start", container("pilosa1")); err != nil {
@ -133,7 +201,11 @@ func TestClusterStuff(t *testing.T) {
 		}

 		client := http.Client{}
-		if req, err := http.NewRequest(http.MethodDelete, "http://pilosa1:10101/index/testidx", nil); err != nil {
+		req, err := http.NewRequest(http.MethodDelete, "http://pilosa1:10101/index/testidx", nil)
+		if auth {
+			req.Header.Set("Authorization", "Bearer "+token)
+		}
+		if err != nil {
 			t.Fatalf("getting req: %v", err)
 		} else if resp, err := client.Do(req); err != nil {
 			t.Fatalf("doing request: %v", err)
@ -146,8 +218,14 @@ func TestClusterStuff(t *testing.T) {
 		}

 		var restoreCmd *exec.Cmd
-		if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101"); err != nil {
-			t.Fatalf("starting restore: %v", err)
+		if auth {
+			if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101", "--auth-token", token); err != nil {
+				t.Fatalf("starting restore: %v", err)
+			}
+		} else {
+			if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101"); err != nil {
+				t.Fatalf("starting restore: %v", err)
+			}
 		}
 		time.Sleep(time.Millisecond * 50)
 		if err = sendCmd("docker", "stop", container("pilosa2")); err != nil {
@ -165,9 +243,16 @@ func TestClusterStuff(t *testing.T) {
 		// now do backup with all nodes down and too short a timeout
 		// so it fails. Has be to be all 3 because the cluster has
 		// replicas=3 and the backup command will retry on replicas.
-		if backupCmd, err = startCmd(
-			"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms"); err != nil {
-			t.Fatalf("sending second backup command: %v", err)
+		if auth {
+			if backupCmd, err = startCmd(
+				"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms", "--auth-token", token); err != nil {
+				t.Fatalf("sending second backup command: %v", err)
+			}
+		} else {
+			if backupCmd, err = startCmd(
+				"featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms"); err != nil {
+				t.Fatalf("sending second backup command: %v", err)
+			}
 		}
 		time.Sleep(time.Millisecond * 10) // want the backup to get started, then fail
 		if err = sendCmd("docker", "stop", container("pilosa1")); err != nil {
@ -198,11 +283,11 @@ func TestClusterStuff(t *testing.T) {
 	})
 }

-func waitForStatus(t *testing.T, stator func(context.Context) (string, error), status string, n int, sleep time.Duration) {
+func waitForStatus(t *testing.T, stator func(context.Context) (string, error), status string, n int, sleep time.Duration, ctx context.Context) {
 	t.Helper()

 	for i := 0; i < n; i++ {
-		s, err := stator(context.TODO())
+		s, err := stator(ctx)
 		if err != nil {
 			t.Logf("Status (try %d/%d): %v (retrying in %s)", i, n, err, sleep.String())
 		} else {
@ -214,7 +299,7 @@ func waitForStatus(t *testing.T, stator func(context.Context) (string, error), s
 		time.Sleep(sleep)
 	}

-	s, err := stator(context.TODO())
+	s, err := stator(ctx)
 	if err != nil {
 		t.Fatalf("querying status: %v", err)
 	}
--- a/internal/clustertests/docker-compose.yml
+++ b/internal/clustertests/docker-compose.yml
@ -65,6 +65,7 @@ services:
      - ENABLE_PILOSA_CLUSTER_TESTS=1
      - GO111MODULE=on
      - PROJECT=${PROJECT}
+      - ENABLE_AUTH=0
    networks:
      - pilosanet
    volumes:
--- a/internal/clustertests/pause_node_test.go
+++ b/internal/clustertests/pause_node_test.go
@ -270,6 +270,12 @@ func TestPauseReplica(t *testing.T) {
 	if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
 		t.Skip("pilosa cluster tests for replication when a replica is paused are not enabled")
 	}
+
+	auth := false
+	if os.Getenv("ENABLE_AUTH") == "1" {
+		auth = true
+	}
+
 	// configurations for test
 	nodeNames := []string{"pilosa1", "pilosa2", "pilosa3"}
 	nodeToPause := "pilosa3"
@ -289,12 +295,17 @@ func TestPauseReplica(t *testing.T) {
 	uri := uris[0]

 	ctx := context.Background()
+	if auth {
+		token := GetAuthToken(t)
+		ctx = context.WithValue(ctx, "token", "Bearer "+token)
+	}
+
 	ctx, cancel := context.WithCancel(ctx)

 	t.Log("start Client")

 	// first achieve normal cluster status
-	waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second)
+	waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second, ctx)

 	// create keyed index
 	rng := rand.New(rand.NewSource(time.Now().UnixNano()))
@ -354,7 +365,7 @@ func TestPauseReplica(t *testing.T) {
 	t.Logf("successfully end insert: %v", len(ts))

 	// wait for cluster status to be non-normal
-	waitForStatus(t, cli.Status, string(disco.ClusterStateDegraded), 30, 1*time.Second)
+	waitForStatus(t, cli.Status, string(disco.ClusterStateDegraded), 30, 1*time.Second, ctx)

 	// wait for cluster status to get back to normal
 	t.Logf("unpause %s", nodeToPause)
@ -362,7 +373,7 @@ func TestPauseReplica(t *testing.T) {
 	if err != nil {
 		t.Fatalf("error on unpause node %s: %v", nodeToPause, err)
 	}
-	waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second)
+	waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second, ctx)

 	// set up directory to store keys
 	basePath := "."