From 0e1cf5bbbddf18157fb09355ed8e405dc6ae8817 Mon Sep 17 00:00:00 2001 From: Souhaila Noor Date: Wed, 26 Jan 2022 17:30:26 -0600 Subject: [PATCH] Enable authentication/authorization for featurebase tools - Add auth-token for featurebase import, backup and restore - Add auth-token to http request - Create a cluster tests with auth enabled - Add test for import with auth enabled --- .gitlab/.gitlab-ci.yml | 1 + Makefile | 8 + api.go | 2 +- client.go | 5 + client/client.go | 34 +- cmd/backup.go | 1 + cmd/import.go | 3 +- cmd/restore.go | 1 + ctl/backup.go | 6 + ctl/import.go | 6 + ctl/import_test.go | 160 +++++++- ctl/restore.go | 38 +- ctl/testdata/certs/README.md | 12 + ctl/testdata/certs/localhost.crt | 25 ++ ctl/testdata/certs/localhost.csr | 16 + ctl/testdata/certs/localhost.key | 27 ++ ctl/testdata/certs/pilosa-ca.crl | 16 + ctl/testdata/certs/pilosa-ca.crt | 29 ++ ctl/testdata/certs/pilosa-ca.key | 51 +++ ctl/testdata/permissions.yaml | 4 + holder.go | 4 +- http/client.go | 108 ++++- http/handler.go | 8 +- internal/authclustertests/docker-compose.yml | 77 ++++ .../authclustertests/testdata/certs/README.md | 12 + .../testdata/certs/localhost.crt | 25 ++ .../testdata/certs/localhost.csr | 16 + .../testdata/certs/localhost.key | 27 ++ .../testdata/certs/pilosa-ca.crl | 16 + .../testdata/certs/pilosa-ca.crt | 29 ++ .../testdata/certs/pilosa-ca.key | 51 +++ .../testdata/featurebase.conf | 383 ++++++++++++++++++ .../testdata/permissions.yaml | 4 + internal/clustertests/cluster_test.go | 121 +++++- internal/clustertests/docker-compose.yml | 1 + internal/clustertests/pause_node_test.go | 17 +- 36 files changed, 1294 insertions(+), 50 deletions(-) create mode 100644 ctl/testdata/certs/README.md create mode 100644 ctl/testdata/certs/localhost.crt create mode 100644 ctl/testdata/certs/localhost.csr create mode 100644 ctl/testdata/certs/localhost.key create mode 100644 ctl/testdata/certs/pilosa-ca.crl create mode 100644 ctl/testdata/certs/pilosa-ca.crt create mode 100644 ctl/testdata/certs/pilosa-ca.key create mode 100644 ctl/testdata/permissions.yaml create mode 100644 internal/authclustertests/docker-compose.yml create mode 100644 internal/authclustertests/testdata/certs/README.md create mode 100644 internal/authclustertests/testdata/certs/localhost.crt create mode 100644 internal/authclustertests/testdata/certs/localhost.csr create mode 100644 internal/authclustertests/testdata/certs/localhost.key create mode 100644 internal/authclustertests/testdata/certs/pilosa-ca.crl create mode 100644 internal/authclustertests/testdata/certs/pilosa-ca.crt create mode 100644 internal/authclustertests/testdata/certs/pilosa-ca.key create mode 100644 internal/authclustertests/testdata/featurebase.conf create mode 100644 internal/authclustertests/testdata/permissions.yaml diff --git a/.gitlab/.gitlab-ci.yml b/.gitlab/.gitlab-ci.yml index 00d61373c..6eac3d465 100644 --- a/.gitlab/.gitlab-ci.yml +++ b/.gitlab/.gitlab-ci.yml @@ -268,6 +268,7 @@ clustertests: - if: '$CI_PIPELINE_SOURCE == "push" || $CI_PIPELINE_SOURCE == "schedule" || $CI_PIPELINE_SOURCE == "web"' script: - make clustertests + - make authclustertests external lookup tests: stage: integration diff --git a/Makefile b/Makefile index 00fc81c8e..5ee669d17 100644 --- a/Makefile +++ b/Makefile @@ -158,6 +158,14 @@ clustertests: vendor PROJECT=$(PROJECT) $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml run client1 $(DOCKER_COMPOSE) -f internal/clustertests/docker-compose.yml down +# Run the cluster tests with authentication enabled +DOCKER_COMPOSE_AUTH = docker-compose -p authclustertests +authclustertests: vendor + $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml down + $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml build + $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml up -d pilosa1 pilosa2 pilosa3 + $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml run client1 + $(DOCKER_COMPOSE_AUTH) -f internal/authclustertests/docker-compose.yml down # Install Pilosa install: diff --git a/api.go b/api.go index 45494b572..5766701a9 100644 --- a/api.go +++ b/api.go @@ -247,7 +247,7 @@ func (api *API) CreateIndex(ctx context.Context, indexName string, options Index } // Create index. - index, err := api.holder.CreateIndexAndBroadcast(cim) + index, err := api.holder.CreateIndexAndBroadcast(ctx, cim) if err != nil { return nil, errors.Wrap(err, "creating index") } diff --git a/client.go b/client.go index 35e3230de..4f0f40ecd 100644 --- a/client.go +++ b/client.go @@ -69,6 +69,7 @@ type InternalClient interface { IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) + IDAllocDataWriter(ctx context.Context, f io.Reader, primary *topology.Node) error IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) FieldTranslateDataReader(ctx context.Context, index, field string) (io.ReadCloser, error) @@ -223,6 +224,10 @@ func (n nopInternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser return nil, nil } +func (n nopInternalClient) IDAllocDataWriter(cctx context.Context, f io.Reader, primary *topology.Node) error { + return nil +} + func (n nopInternalClient) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) { return nil, nil } diff --git a/client/client.go b/client/client.go index bc5740514..19d532e50 100644 --- a/client/client.go +++ b/client/client.go @@ -61,6 +61,8 @@ type Client struct { shardNodes shardNodes tick *time.Ticker done chan struct{} + + AuthToken string } func (c *Client) getURIsForShard(index string, shard uint64) ([]*pnet.URI, error) { @@ -283,7 +285,7 @@ func (c *Client) Query(query PQLQuery, options ...interface{}) (*QueryResponse, return nil, errors.Wrap(err, "making request data") } path := fmt.Sprintf("/index/%s/query", query.Index().name) - _, respData, err := c.HTTPRequest("POST", path, reqData, defaultProtobufHeaders()) + _, respData, err := c.HTTPRequest("POST", path, reqData, c.augmentHeaders(defaultProtobufHeaders())) if err != nil { return nil, err } @@ -306,7 +308,7 @@ func (c *Client) CreateIndex(index *Index) error { data := []byte(index.options.String()) path := fmt.Sprintf("/index/%s", index.name) - status, body, err := c.HTTPRequest("POST", path, data, nil) + status, body, err := c.HTTPRequest("POST", path, data, c.augmentHeaders(nil)) if err != nil { return errors.Wrapf(err, "creating index: %s", index.name) } @@ -330,7 +332,7 @@ func (c *Client) CreateField(field *Field) error { data := []byte(field.options.String()) path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name) - status, body, err := c.HTTPRequest("POST", path, data, nil) + status, body, err := c.HTTPRequest("POST", path, data, c.augmentHeaders(nil)) if err != nil { return errors.Wrapf(err, "creating field: %s in index: %s", field.name, field.index.name) } @@ -398,7 +400,7 @@ func (c *Client) DeleteIndexByName(index string) error { defer span.Finish() path := fmt.Sprintf("/index/%s", index) - _, _, err := c.HTTPRequest("DELETE", path, nil, nil) + _, _, err := c.HTTPRequest("DELETE", path, nil, c.augmentHeaders(nil)) return err } @@ -408,7 +410,7 @@ func (c *Client) DeleteField(field *Field) error { defer span.Finish() path := fmt.Sprintf("/index/%s/field/%s", field.index.name, field.name) - _, _, err := c.HTTPRequest("DELETE", path, nil, nil) + _, _, err := c.HTTPRequest("DELETE", path, nil, c.augmentHeaders(nil)) return err } @@ -597,7 +599,7 @@ func (c *Client) fetchFragmentNodes(indexName string, shard uint64) ([]fragmentN return []fragmentNode{*c.manualFragmentNode}, nil } path := fmt.Sprintf("/internal/fragment/nodes?shard=%d&index=%s", shard, indexName) - _, body, err := c.HTTPRequest("GET", path, []byte{}, nil) + _, body, err := c.HTTPRequest("GET", path, []byte{}, c.augmentHeaders(nil)) if err != nil { return nil, err } @@ -635,7 +637,7 @@ func (c *Client) fetchPrimaryNode() (fragmentNode, error) { } func (c *Client) importData(uri *pnet.URI, path string, data []byte) error { - if status, _, err := c.doRequest(uri, "POST", path, defaultProtobufHeaders(), data); err != nil { + if status, _, err := c.doRequest(uri, "POST", path, c.augmentHeaders(defaultProtobufHeaders()), data); err != nil { return errors.Wrapf(err, "import to %s", uri.HostPort()) } else if status == http.StatusPreconditionFailed { return ErrPreconditionFailed @@ -683,7 +685,8 @@ func (c *Client) importRoaringBitmap(uri *pnet.URI, field *Field, shard uint64, return err } - status, _, err := c.doRequest(uri, "POST", path, defaultProtobufHeaders(), data) + header := c.augmentHeaders(defaultProtobufHeaders()) + status, _, err := c.doRequest(uri, "POST", path, header, data) if err != nil { return errors.Wrapf(err, "roaring import to %s, status: %d", uri.HostPort(), status) } @@ -724,7 +727,7 @@ func (c *Client) Info() (Info, error) { span := c.tracer.StartSpan("Client.Info") defer span.Finish() - _, data, err := c.HTTPRequest("GET", "/info", nil, nil) + _, data, err := c.HTTPRequest("GET", "/info", nil, c.augmentHeaders(nil)) if err != nil { return Info{}, errors.Wrap(err, "requesting /info") } @@ -754,7 +757,7 @@ func (c *Client) Status() (Status, error) { } func (c *Client) readSchema() ([]SchemaIndex, error) { - _, data, err := c.HTTPRequest("GET", "/schema", nil, nil) + _, data, err := c.HTTPRequest("GET", "/schema", nil, c.augmentHeaders(nil)) if err != nil { return nil, errors.Wrap(err, "requesting /schema") } @@ -1021,6 +1024,9 @@ func (c *Client) augmentHeaders(headers map[string]string) map[string]string { version := strings.TrimPrefix(Version, "v") headers["User-Agent"] = fmt.Sprintf("pilosa/client/%s", version) + if c.AuthToken != "" { + headers["Authorization"] = c.AuthToken + } return headers } @@ -1177,7 +1183,7 @@ func (c *Client) startTransaction(id string, timeout time.Duration, exclusive bo return nil, errors.Wrap(err, "marshalling transaction") } - status, data, err := c.httpRequest("POST", "/transaction", bod, defaultJSONHeaders(), true) + status, data, err := c.httpRequest("POST", "/transaction", bod, c.augmentHeaders(defaultJSONHeaders()), true) if status == http.StatusConflict && time.Now().Before(deadline) { // if we're getting StatusConflict after all the usual timeouts/retries, keep retrying until the deadline time.Sleep(time.Second) @@ -1204,7 +1210,7 @@ func (c *Client) startTransaction(id string, timeout time.Duration, exclusive bo } func (c *Client) FinishTransaction(id string) (*pilosa.Transaction, error) { - _, data, err := c.httpRequest("POST", "/transaction/"+id+"/finish", nil, defaultJSONHeaders(), true) + _, data, err := c.httpRequest("POST", "/transaction/"+id+"/finish", nil, c.augmentHeaders(defaultJSONHeaders()), true) if err != nil && len(data) == 0 { return nil, err } @@ -1226,7 +1232,7 @@ func (c *Client) FinishTransaction(id string) (*pilosa.Transaction, error) { } func (c *Client) Transactions() (map[string]*pilosa.Transaction, error) { - _, respData, err := c.httpRequest("GET", "/transactions", nil, defaultJSONHeaders(), true) + _, respData, err := c.httpRequest("GET", "/transactions", nil, c.augmentHeaders(defaultJSONHeaders()), true) if err != nil { return nil, errors.Wrap(err, "getting transactions") } @@ -1240,7 +1246,7 @@ func (c *Client) Transactions() (map[string]*pilosa.Transaction, error) { } func (c *Client) GetTransaction(id string) (*pilosa.Transaction, error) { - _, data, err := c.httpRequest("GET", "/transaction/"+id, nil, defaultJSONHeaders(), true) + _, data, err := c.httpRequest("GET", "/transaction/"+id, nil, c.augmentHeaders(defaultJSONHeaders()), true) if err != nil { return nil, err } diff --git a/cmd/backup.go b/cmd/backup.go index 0a0d5dd28..8166d7bd2 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -31,5 +31,6 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file. flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.") flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.") ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification) + flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token") return ccmd } diff --git a/cmd/import.go b/cmd/import.go index 243f440e8..d3f50dd27 100644 --- a/cmd/import.go +++ b/cmd/import.go @@ -5,7 +5,7 @@ import ( "context" "io" - "github.com/molecula/featurebase/v3" + pilosa "github.com/molecula/featurebase/v3" "github.com/molecula/featurebase/v3/ctl" "github.com/spf13/cobra" ) @@ -51,6 +51,7 @@ omitted. If it is present then its format should be YYYY-MM-DDTHH:MM. flags.BoolVarP(&Importer.CreateSchema, "create", "e", false, "Create the schema if it does not exist before import.") flags.BoolVarP(&Importer.Clear, "clear", "", false, "Clear the data provided in the import.") ctl.SetTLSConfig(flags, "", &Importer.TLS.CertificatePath, &Importer.TLS.CertificateKeyPath, &Importer.TLS.CACertPath, &Importer.TLS.SkipVerify, &Importer.TLS.EnableClientVerification) + flags.StringVar(&Importer.AuthToken, "auth-token", "", "Authentication token") return importCmd } diff --git a/cmd/restore.go b/cmd/restore.go index 071463714..f9f8c32db 100644 --- a/cmd/restore.go +++ b/cmd/restore.go @@ -27,6 +27,7 @@ The Restore command will take a backup archive and restore it to a new, clean cl flags.IntVar(&cmd.Concurrency, "concurrency", 1, "number of concurrent uploads") flags.DurationVar(&cmd.RetryPeriod, "retry-period", cmd.RetryPeriod, "Length of time after HTTP request failure to continue retrying request.") flags.StringVar(&cmd.Pprof, "pprof", cmd.Pprof, "host:port to listen for profiling requests at /debug/pprof and /debug/fgprof.") + flags.StringVar(&cmd.AuthToken, "auth-token", "", "Authentication token") ctl.SetTLSConfig( flags, "", &cmd.TLS.CertificatePath, diff --git a/ctl/backup.go b/ctl/backup.go index 8d3aa4e1a..ff5a93307 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -52,6 +52,8 @@ type BackupCommand struct { // nolint: maligned *pilosa.CmdIO TLS server.TLSConfig + + AuthToken string } // NewBackupCommand returns a new instance of BackupCommand. @@ -93,6 +95,10 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) { } cmd.client = client + if cmd.AuthToken != "" { + ctx = context.WithValue(ctx, "token", "Bearer "+cmd.AuthToken) + } + // Determine the field type in order to correctly handle the input data. indexes, err := cmd.client.Schema(ctx) if err != nil { diff --git a/ctl/import.go b/ctl/import.go index 34689ec9c..3b18b8499 100644 --- a/ctl/import.go +++ b/ctl/import.go @@ -54,6 +54,8 @@ type ImportCommand struct { // nolint: maligned *pilosa.CmdIO TLS server.TLSConfig + + AuthToken string } // NewImportCommand returns a new instance of ImportCommand. @@ -84,6 +86,10 @@ func (cmd *ImportCommand) Run(ctx context.Context) error { } cmd.client = client + if cmd.AuthToken != "" { + ctx = context.WithValue(ctx, "token", "Bearer "+cmd.AuthToken) + } + if cmd.CreateSchema { if cmd.FieldOptions.Type == "" { // set the correct type for the field diff --git a/ctl/import_test.go b/ctl/import_test.go index b9327bb9b..0511100db 100644 --- a/ctl/import_test.go +++ b/ctl/import_test.go @@ -9,12 +9,17 @@ import ( "io" "io/ioutil" "net/http" + "os" "reflect" "strings" "testing" "time" - "github.com/molecula/featurebase/v3" + "github.com/golang-jwt/jwt" + pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/authn" + "github.com/molecula/featurebase/v3/logger" + "github.com/molecula/featurebase/v3/server" "github.com/molecula/featurebase/v3/test" "github.com/molecula/featurebase/v3/testhook" ) @@ -568,3 +573,156 @@ func TestImportCommand_RunBool(t *testing.T) { } }) } + +func TestImport_AuthOn(t *testing.T) { + clusterSize := 1 + + logFilename := "./testdata/query.log" + _, err := os.Create(logFilename) + if err != nil { + t.Fatalf("Failed to create query log file: %s", err) + } + + auth := server.Auth{ + Enable: true, + ClientId: "e9088663-eb08-41d7-8f65-efb5f54bbb71", + ClientSecret: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF", + AuthorizeURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize", + TokenURL: "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token", + GroupEndpointURL: "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true", + LogoutURL: "https://login.microsoftonline.com/common/oauth2/v2.0/logout", + Scopes: []string{"https://graph.microsoft.com/.default", "offline_access"}, + SecretKey: "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF", + RedirectBaseURL: "https://localhost:0", + QueryLogPath: logFilename, + PermissionsFile: "./testdata/permissions.yaml", + } + + commandOpts := make([][]server.CommandOption, clusterSize) + configs := make([]*server.Config, clusterSize) + for i := range configs { + conf := server.NewConfig() + configs[i] = conf + conf.Bind = "https://localhost:0" + conf.Auth = auth + conf.TLS.CertificatePath = "./testdata/certs/localhost.crt" + conf.TLS.CertificateKeyPath = "./testdata/certs/localhost.key" + conf.TLS.CACertPath = "./testdata/certs/pilosa-ca.crt" + conf.TLS.EnableClientVerification = false + conf.TLS.SkipVerify = true + commandOpts[i] = append(commandOpts[i], server.OptCommandConfig(conf)) + } + + a, err := authn.NewAuth( + logger.NewStandardLogger(os.Stdout), + "http://localhost:0/", + auth.Scopes, + auth.AuthorizeURL, + auth.TokenURL, + auth.GroupEndpointURL, + auth.LogoutURL, + auth.ClientId, + auth.ClientSecret, + auth.SecretKey, + ) + if err != nil { + t.Fatal(err) + } + + // make a valid token + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "group-id-test", GroupName: "group-name-test"}}) + claims["molecula-idp-groups"] = groupString + claims["oid"] = "42" + claims["name"] = "valid" + token, err := tkn.SignedString([]byte(a.SecretKey())) + if err != nil { + t.Fatal(err) + } + validToken := "Bearer " + token + invalidToken := "Bearer " + string(tkn.Raw) + + tests := []struct { + Index string + Field string + CreateSchema bool + Token string + Err error + }{ + { + Index: "test", + Field: "field1", + CreateSchema: true, + Token: validToken, + Err: nil, + }, + { + Index: "test", + Field: "field1", + CreateSchema: false, + Token: validToken, + Err: nil, + }, + { + Index: "test", + Field: "field1", + CreateSchema: false, + Token: invalidToken, + Err: fmt.Errorf("token contains an invalid number of segments"), + }, + { + Index: "test", + Field: "field1", + CreateSchema: true, + Token: invalidToken, + Err: fmt.Errorf("token contains an invalid number of segments"), + }, + } + + t.Run("set", func(t *testing.T) { + buf := bytes.Buffer{} + stdin, stdout, stderr := GetIO(buf) + cm := NewImportCommand(stdin, stdout, stderr) + file, err := testhook.TempFile(t, "import.csv") + if err != nil { + t.Fatalf("creating tempfile: %v", err) + } + _, err = file.Write([]byte("1,2\n3,4\n5,6")) + if err != nil { + t.Fatalf("writing to tempfile: %v", err) + } + + if err != nil { + t.Fatal(err) + } + + cluster := test.MustRunCluster(t, clusterSize, commandOpts...) + defer cluster.Close() + cmd := cluster.GetNode(0) + cm.Host = cmd.API.Node().URI.HostPort() + + for i, test := range tests { + cm.Index = test.Index + cm.Field = test.Field + cm.CreateSchema = test.CreateSchema + cm.Paths = []string{file.Name()} + ctx := context.WithValue(context.Background(), "token", test.Token) + err = cm.Run(ctx) + if test.Err != nil { + if !strings.Contains(err.Error(), test.Err.Error()) { + t.Fatalf("Test: %d, Import Run doesn't work: got %s, expected: %s", i, err, test.Err) + } + } else { + if err != test.Err { + t.Fatalf("Test: %d, Import Run doesn't work: got %s, expected: %s", i, err, test.Err) + } + } + + } + err = os.Remove(logFilename) + if err != nil { + t.Fatalf("Failed to delete query log file: %s", err) + } + }) +} diff --git a/ctl/restore.go b/ctl/restore.go index b40b25b78..8f370ba0e 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -29,7 +29,8 @@ import ( // RestoreCommand represents a command for restoring a backup to type RestoreCommand struct { tlsConfig *tls.Config - Host string + + Host string Concurrency int @@ -47,7 +48,10 @@ type RestoreCommand struct { // Standard input/output *pilosa.CmdIO + TLS server.TLSConfig + + AuthToken string } // NewRestoreCommand returns a new instance of RestoreCommand. @@ -88,6 +92,10 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) { } cmd.client = client + if cmd.AuthToken != "" { + ctx = context.WithValue(ctx, "token", "Bearer "+cmd.AuthToken) + } + nodes, err := cmd.client.Nodes(ctx) if err != nil { return err @@ -139,8 +147,24 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology. if len(existingSchema) == 0 { cmd.Logger().Printf("Load Schema") url := primary.URI.Path("/schema") + req, err := retryablehttp.NewRequest("POST", url, f) + if err != nil { + return err + } + req = req.WithContext(ctx) + req.Header.Add("Accept", "application/json") + + token, ok := ctx.Value("token").(string) + if ok && token != "" { + req.Header.Set("Authorization", token) + } + client := cmd.newClient() - _, err = client.Post(url, "application/json", f) + _, err = client.Do(req) + if err != nil { + return err + } + } else { schema := &pilosa.Schema{} if err := json.NewDecoder(f).Decode(schema); err != nil { @@ -222,10 +246,9 @@ func (cmd *RestoreCommand) restoreIDAlloc(ctx context.Context, primary *topology defer f.Close() logger.Printf("Load idalloc") - url := primary.URI.Path("/internal/idalloc/restore") - client := cmd.newClient() - _, err = client.Post(url, "application/octet-stream", f) + err = cmd.client.IDAllocDataWriter(ctx, f, primary) + return err } @@ -301,6 +324,11 @@ func (cmd *RestoreCommand) restoreShard(ctx context.Context, filename string) er req = req.WithContext(ctx) req.Header.Set("Content-Type", "application/octet-stream") + token, ok := ctx.Value("token").(string) + if ok && token != "" { + req.Header.Set("Authorization", token) + } + client := cmd.newClient() resp, err := client.Do(req) if err != nil { diff --git a/ctl/testdata/certs/README.md b/ctl/testdata/certs/README.md new file mode 100644 index 000000000..4c1009a9c --- /dev/null +++ b/ctl/testdata/certs/README.md @@ -0,0 +1,12 @@ + + +# these test certs were generated with the following commands + +certstrap --depot-path certs init --common-name pilosa-ca --expires "100 years" +certstrap --depot-path certs request-cert --common-name localhost --domain localhost +certstrap --depot-path certs sign "localhost" --CA pilosa-ca --expires "100 years" + +# certstrap version +dev-25ea708a + +(built with go 1.13) diff --git a/ctl/testdata/certs/localhost.crt b/ctl/testdata/certs/localhost.crt new file mode 100644 index 000000000..8269ccda6 --- /dev/null +++ b/ctl/testdata/certs/localhost.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEPjCCAiagAwIBAgIRAJ7rl74WPv8pLuhVRXt6fV0wDQYJKoZIhvcNAQELBQAw +FDESMBAGA1UEAxMJcGlsb3NhLWNhMCAXDTIwMTAyMDE5MTMzNFoYDzIxMjAxMDIw +MTkxMzE5WjAUMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQDmi8FMWt23M0Cr2aCgEXUGQ0gv/4M7CXH/5GkSI866YwGV +Bd1iZMBRiONQwvGDnqYZRrAQv6mFjfyBqxdkbh++74FC3JK7sLhks0vg5VwbHV7T +5kj3bJqd+LKn5qPPOQXX9sgmv/NkggF/XXwF73noLPmgDQ78S+OP0ANmi1TQiU3a +gE+qp+Qpl5KC7dH9aC9nvE9iGfEcGNr+rXj05liiXqe4ZtIKWjeke7Ej64C6qX97 +bNPzmLARtqbRsIkfAU8SJy3YuHfW8n1xr4B7ENm9jHQCh1wUv2YhaPpnio+/R2zp +Lw4yCqilDX9ZZ4nG3cBFuziSf+BUXJ9ydbw1aX8HAgMBAAGjgYgwgYUwDgYDVR0P +AQH/BAQDAgO4MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAdBgNVHQ4E +FgQUJRQJpaR5bp4ZyUsMxgK+UJl6PSgwHwYDVR0jBBgwFoAU69lmXSa5BZeyYU/6 +XpWdtr59H1YwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IC +AQChxsBZ/b14ukJXX48BxAyZcy5r7GrLcRGQ3guUTONFVDWPzzpd8mjHi0yJDhMW +2zWtw3/H+c+zT7uRd+2sUxFdpAurNSFCdV++5Q/0aFvl+By5+MhVhtznEQDU0/lM +zFxiEYe/N9Vi2N0S1KPxvYL/RfBU27u+O/50zhjueM1BTyHTTqL6E2DFeT2VPKIg +zCDUtiTEDFZrD0XGITT/3CIoNCK8aC+Fq65OEoyEn6qR5qg1Kc4tfZmo6hWYiSlR +XeP36cP9R8kEMte1BdE74GVqE9cTuVZERdgB0hv3EME7Byq7uIm/a+JXbsh2/OFm +HcE0/HP+O0YK8YaVMGwI3pZYy2syWqPcakcvusETehr6P+Ihh2cOKRwqkCl6b87e +uSLJNTUMKZgakW6Bjv6lgQaWqnKzTC/RgmQ+G3w0nKATX9+jYE2j3MzZhbtcml+2 +gp6u225yAJaYt/MQidwUMiKYeCgjaUNoL0fOJesGkokPk80ceISnqvbSRiZRTvK1 +bVenkhkBrHuvvgKVstzcuZI9oQ2snWhK1naVQiOtQNEFUCHwyU95zADOK0km88NB +2het6yYaEUL9csHPEjPd3lFglerGQnil2Ly1slUC4jb7hfVRHjOFs8PVr9gQ45dW +Jvsv4pawHKFE0ennoNvoDmzbiY1TY5ScTZquPGIsEBV+tQ== +-----END CERTIFICATE----- diff --git a/ctl/testdata/certs/localhost.csr b/ctl/testdata/certs/localhost.csr new file mode 100644 index 000000000..1814b72af --- /dev/null +++ b/ctl/testdata/certs/localhost.csr @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICgDCCAWgCAQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA5ovBTFrdtzNAq9mgoBF1BkNIL/+DOwlx/+RpEiPO +umMBlQXdYmTAUYjjUMLxg56mGUawEL+phY38gasXZG4fvu+BQtySu7C4ZLNL4OVc +Gx1e0+ZI92yanfiyp+ajzzkF1/bIJr/zZIIBf118Be956Cz5oA0O/Evjj9ADZotU +0IlN2oBPqqfkKZeSgu3R/WgvZ7xPYhnxHBja/q149OZYol6nuGbSClo3pHuxI+uA +uql/e2zT85iwEbam0bCJHwFPEict2Lh31vJ9ca+AexDZvYx0AodcFL9mIWj6Z4qP +v0ds6S8OMgqopQ1/WWeJxt3ARbs4kn/gVFyfcnW8NWl/BwIDAQABoCcwJQYJKoZI +hvcNAQkOMRgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQAD +ggEBABMi2/4j1/qzwWAYlEs2KW3z+apzzDLKgjE0kY6QvELh/8aBj0rMglb0HM2x +4iSSoX1ZwZgDZ9fIJ3klG/UF7CUweMghb9yC2PP9Z8WuqaECQyM87KgSln8PND9E +1OvD30rp9yr9KxEeckq+c1ebLi/qGrIY21VCwfxA0mv3sfi7Q5ONIckay/Xj+1Tz +ovE/TkM/8wTE/SKbpQSCkP7K1NDXuAhMGjcN0x3d3f8nBcLcZOrRroiHy38Bv/9T +Vd62IY6uqYw9sluBbMX72D/mmJiCKEw3+DhDJFhHCTCrAQM0QwLuwnG2lQFYoENc +ZAkwDIi+3DXHEEyloNSYGtXMEiA= +-----END CERTIFICATE REQUEST----- diff --git a/ctl/testdata/certs/localhost.key b/ctl/testdata/certs/localhost.key new file mode 100644 index 000000000..b7434fdc9 --- /dev/null +++ b/ctl/testdata/certs/localhost.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA5ovBTFrdtzNAq9mgoBF1BkNIL/+DOwlx/+RpEiPOumMBlQXd +YmTAUYjjUMLxg56mGUawEL+phY38gasXZG4fvu+BQtySu7C4ZLNL4OVcGx1e0+ZI +92yanfiyp+ajzzkF1/bIJr/zZIIBf118Be956Cz5oA0O/Evjj9ADZotU0IlN2oBP +qqfkKZeSgu3R/WgvZ7xPYhnxHBja/q149OZYol6nuGbSClo3pHuxI+uAuql/e2zT +85iwEbam0bCJHwFPEict2Lh31vJ9ca+AexDZvYx0AodcFL9mIWj6Z4qPv0ds6S8O +MgqopQ1/WWeJxt3ARbs4kn/gVFyfcnW8NWl/BwIDAQABAoIBAFX+GPqfBgY4cs3m +3ff2qvzMCdgFaXCS5Fe7XcmrW4fAOC3awynZRLbk5U0Reb5LZc8Vw8RriRLM1DuV +kqMeRG8WrNNArOafUxgUnJ/lTUa73MwTIHJRqxZzVkg0SjOYJGranOt/O4zoxSA5 +wXIBUipc5Dtjw4wtzlKtFyefnuItL2MCdwOHUdZfnhr9Oykp1fuNqBqkkeryj3XV +ukHQvqU5zkMSayprNglziqTHUzU33iyZeDng+CJQeYTEc7Gn+zja2SFFBlPHqXXo +/OzAr94zI3vOnj3yRM3+sKMJVPV+RoJEGpsvPVuVn38d1VnIMEx8Gy/wif6tmM9c +7Q44hKECgYEA/JMFwkPGbry80ktDI065k5FIYn1EDRyUaQqyskmkBRcNW3qOShqj +o/zWQfCgxP587IEdKBBwqCpdqfghi3EW+JqfVlbGY6t1chAurYF/47CTIgKO5qRM +GdCY2OdiAeo5nba/KiLQfSuY08MCNDrQabLRJXIng8qVWpRwQzsv5rECgYEA6aw/ +HugeQhTxk2uV91jJaAQIaxrt6JxuoG0CGGlbDrrTl2dnbPYA0muHMFdT/bzKjCpv +n/ScqbCyHuy+lWSnOzgedRNQCB46+0H58LAjITAj9QaT3raZqVReVIaD+pnx27dp +Cw5Ws6ENa9AQey3DO+dkRWot2AcLSw6TGR8HnzcCgYEA/ArfCU/G2cSgDJ6sPbSW +vaqR+C6W1Rq7AuN5FS8lbSrm2m2/RjW1LLTnPmAYntxx3zSs2sklErtMQovpNZRB +3w21iVwIl3eHOK7rVZtP+u++s4aoAYLcqjod/P1RMSYCHt85fpvFP9Ncq50DOwmh +5ohZ6ysyQXLMfdp4+K48i9ECgYEApOFALKu2ZgRnLRFV4REKFFX8Jq76vg5bVOF2 +AAmfEbasBIIXDWBL1i2/V1HXVwv2k46B8wjj3ixqkr2UAM/j3DpN62g0KXZDQfUc +ykNOlmVkickZX6XSqRN5+ARubc5gRRuWiBGXBeqXEMLgTjpNLyCntP8l1++ofU6M +ZsZpV2MCgYA3nfNXAR5O4B/dm/2HmDQrXy0qia7Hwi/95pgL2FJaEmjBCPI1j12o +M5YCbhpr1pwsNKPV9AUlUz+OCwS8Vt+V0gQf9/XvNOsifU+mbMYVpuNGwmcKafnv +qECSeidrmhWJSR/SSNBcE94im/8ObVU110WJMkjC9otjDl9Aua/3LQ== +-----END RSA PRIVATE KEY----- diff --git a/ctl/testdata/certs/pilosa-ca.crl b/ctl/testdata/certs/pilosa-ca.crl new file mode 100644 index 000000000..3b25dd052 --- /dev/null +++ b/ctl/testdata/certs/pilosa-ca.crl @@ -0,0 +1,16 @@ +-----BEGIN X509 CRL----- +MIIChTBvAgEBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMTCXBpbG9zYS1jYRcN +MjAxMDIwMTkxMzIyWhgPMjEyMDEwMjAxOTEzMjBaMACgIzAhMB8GA1UdIwQYMBaA +FOvZZl0muQWXsmFP+l6Vnba+fR9WMA0GCSqGSIb3DQEBCwUAA4ICAQBja+EDQAp+ +KeD7UhWMMrTd9j03GgQ2E2Z7+Ba0qJ5+kS7/t+Yja2o5dQJkrC3GwEMOQb6DRRUE +nUE4xlr5Rryoq0dZk+Lp1f4cHrnP8l1xylUL44gsnY4v8zMR8L8X98vj7kKCqB8w +DFX7qkMlE5Ie2Hha7uuOJ85FnxIbMcRxFQH2m2zDfWG8/Lmxezvv9Hn45V/kwIQy +MmBh6cNuhzEneyNpM9yMRe/29QgVitF/2q6d+FzK8w8hkUFeYlyM+cP7F4Ml7160 +UidSQM04zvBtJ8frZAvrDaPBBZhrTXcyw6+Qnp/aaW1ZsEIdHEcbYGNdgtazleoG +VH35cDP90KfiRbq69PQ9Zqn3cI//MX3sHrglA9wsEhHc9P7dowHaOFyxPouZPEmQ +/Jqg5oyJzujRwhf0v3SdJvhuDEzla2N+QyYRk0kRHtdv+glz7T7CnTYCk+DTv+oh +QABUrCbjfBoE5M2Qep9ZkIbl2gaDCpvbZSF4zFLKQc2aIOBpVn3HgTGBvdFD3FJY +Txl2F4Y3rS1T/WMAH86cZIc9h5HlMdFtAFnHAlHtB3wGw3FD/GcvGcvz2D4GaxKq +erzrnOxjYOA4M0haGzWF6dC7aPA8y35eZuqNXvbenTtc7A11bWTJfG1I7ctvLyPE +MpCNMHfymh/XtYZiZhvu6ueu3OeKScN+tA== +-----END X509 CRL----- diff --git a/ctl/testdata/certs/pilosa-ca.crt b/ctl/testdata/certs/pilosa-ca.crt new file mode 100644 index 000000000..9878e3aa7 --- /dev/null +++ b/ctl/testdata/certs/pilosa-ca.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE6jCCAtKgAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwlwaWxv +c2EtY2EwIBcNMjAxMDIwMTkxMzIyWhgPMjEyMDEwMjAxOTEzMjBaMBQxEjAQBgNV +BAMTCXBpbG9zYS1jYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALT/ +uNmbnfXWNX+FsL0Waqw/5deti5F4cSjMrGRQpXxalTcooqNk/lkeqXkvi9ooFROZ +/HyQR9GM9dSD/aj6gD3FnGA4ueB24Xr6bWsRpDRh6+3UGLB3YCNNdGLSfX3LPMYh +RJutFmsg+r6SrSytbLbffu+0a/4fxtajZNwQJjDjd8qflXQZYlzp2LHk1A/jqqdI +fBtqkNg925TGKiavvUqKtdI/eFzRoiQ7NLBUJmszzveUXvOUMsMnW2/myLBe3Oqk +Vsy85lya0ADln20C3Lb0+ZA4KoGX3EWdtBXEuWqMoyvCJoJ4I3bH2LlfOUjRt8UE +pPk6sPMROJ+75mlgvgnSlYsN8PaZdvdm2VGVWRWUyEfyW/qa2fv8d2XBWqibl0YF +tqay9CX1aWgC9q12yx3vj7Yh+ZNbeZFLc7IL8zyNMwIjIOIIyGBY70KewfgVktzq +fAMz6h1sr9Kxozil97Cu3ma4B6UiL3rUbYMO/rNhVxcIuUoJpgIEVuRt+uXEG74y +XftauZ67qILFQzfpoacncvDEx5nJ3itLgbbyt1n1iWdGuEiMSLFT+x+nNUgVpQgA +sWRYxHdisM4xzRVN6pAaToMs1p8Ju7l9xU3z7RSogTyVk9gMIIV4t9TDYP5fI10Y +GFi7B6q0t3pIGXgKySHjCSl0EKYkDQDFl5tWeaiXAgMBAAGjRTBDMA4GA1UdDwEB +/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTr2WZdJrkFl7Jh +T/pelZ22vn0fVjANBgkqhkiG9w0BAQsFAAOCAgEAnlBFrWhB+WesCc3lhK980rA6 +roNFYMZdaXvg4zaEGergkRvPab5yXoof1AAeznJm45GQfXn8HbQlrZmAqWg3fNld +/TX+jNvosM8K8K+PzesDGHsm/eQnbrb0qzMDsQgFY+nnD+x/ZQtjmKZtcNr/0ZlM +EJeXWU5cGy70GMbNztspMHsOLa3ZDLsBOJYOwSFxDlLDFrjZoRoPCWw8jRL+Tb4t +JjZcGZDD4a5+DqcojanIdNU1yI4teP6aV1LQTVNn4pwOap+tD0De/WzOPmXTQq5M +9ssxL7xSqVShQQMC8LVSWSRxtT6kLq0Av6i7wio0DZGnH3ynERTUs13DRZkwbVsE +OaQLmiQnsHRTIpdts/fswZ2FRPvdhhXxBjiGQZGEGXXznxHTNJ6nQioh95Ft5hNA +82i8Z74miaFIT/33/sZ5SuwUzphCgqCY2x7NUS8J313O9lsar0bweJTvQaZg/69E +PmmwUcDebh+pgKP01z4BqTzhtchmFUKzT+oOC8tmTeSlhsBTzO4xMw6OqhpXKL+c +k9f2CGUZYtEZHDRmP+C++FEi+B/tV2Oq3on+QPiaIIcRsOftthGUvJ8htUl3w+hq +B5TnL8CeLjXGKKRp+UiakrB4E7y2aIbrtIRnJ/Llg2XMND/0xbldsNsyDNXCIoDH +sz8HqwF3CUbv5XD4ioY= +-----END CERTIFICATE----- diff --git a/ctl/testdata/certs/pilosa-ca.key b/ctl/testdata/certs/pilosa-ca.key new file mode 100644 index 000000000..135a6e233 --- /dev/null +++ b/ctl/testdata/certs/pilosa-ca.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAtP+42Zud9dY1f4WwvRZqrD/l162LkXhxKMysZFClfFqVNyii +o2T+WR6peS+L2igVE5n8fJBH0Yz11IP9qPqAPcWcYDi54HbhevptaxGkNGHr7dQY +sHdgI010YtJ9fcs8xiFEm60WayD6vpKtLK1stt9+77Rr/h/G1qNk3BAmMON3yp+V +dBliXOnYseTUD+Oqp0h8G2qQ2D3blMYqJq+9Soq10j94XNGiJDs0sFQmazPO95Re +85Qywydbb+bIsF7c6qRWzLzmXJrQAOWfbQLctvT5kDgqgZfcRZ20FcS5aoyjK8Im +gngjdsfYuV85SNG3xQSk+Tqw8xE4n7vmaWC+CdKViw3w9pl292bZUZVZFZTIR/Jb ++prZ+/x3ZcFaqJuXRgW2prL0JfVpaAL2rXbLHe+PtiH5k1t5kUtzsgvzPI0zAiMg +4gjIYFjvQp7B+BWS3Op8AzPqHWyv0rGjOKX3sK7eZrgHpSIvetRtgw7+s2FXFwi5 +SgmmAgRW5G365cQbvjJd+1q5nruogsVDN+mhpydy8MTHmcneK0uBtvK3WfWJZ0a4 +SIxIsVP7H6c1SBWlCACxZFjEd2KwzjHNFU3qkBpOgyzWnwm7uX3FTfPtFKiBPJWT +2AwghXi31MNg/l8jXRgYWLsHqrS3ekgZeArJIeMJKXQQpiQNAMWXm1Z5qJcCAwEA +AQKCAgAWmjiDNCOtp2pW2mMPudToXbJeFJXxPJEk/yon/MotlUI8+R4WOW5pwqJ3 +N7DHNWosYHZfN8VALdIlD7aFe4K4NA0rFupfVXki2lL/o9xVjkTgFjRfFQk0X1/B +V3fEVbTpKQ5gQmUiS6QEWFy3z5Bb5dz8IhO6UE2MUCswL/QU9tLmwrbvIJxf7fPZ +gzHYKh4NdcfJxK0B0/evxG9PFXMV8+xwrOxi6urMi3gw7NE/YeDemfChikgshqWs +e61kGPSNeKg+OPirZ8nB0urtugXF8yGXGOx18njXWLI8Zayh2Z4mwL/+WvJSyvIN +dA67QTUprULMvL+MGwJvMA+96Q7SBRVKR9HHNaP9pFsZup3QX9mqDQ9miB6+rzn7 +f5RiSLVgq+HUPMPfqgXCkQZBcY28TcM1BZhS4uJJTkbvVTkrlHKJs6Q/LBqJFvq0 +3+2M1xQb4HdRTRlwZ/YsxdqXGIoA3Xx3nZbb6LlPp/MT93xxsLJNNP47n445Cw8i +lz7hJJDwo+TyXmRRWKlFXO8TEhqKhK9ZEXmkXBxSeCQV0oTYS5kU9XdZ5iu5/CQQ +Lv+uFQfHTPWm/Lp5RC8JEEJwK5bwRAs5d9oWg5EbWJp2ol36g3YO1np5RFsQn3jL +qJPz35X3Bp9zQeZcAZpt1fWFdyX9f7V2LLCY8gUyIlfCEXBWwQKCAQEA8Uoes4ph +15tM1LswmOLqOq9iWuvcKYSNcz8my8nlP5zkUdGKvGcLblm5Mg4wKoPFiyjKhX7I +S8DUN8x7E5aiNBiZ0PGku4CKjubQgdFG/rYRfnrUEaA81Uw3a69eYioKbxhhxDzb +Lqd0/tGNHQZBQEyofOChwqTWpAdIh79F1oXejcPzRDr+oXxJVnvJDdYCEN/0TYkz +qeJdEtnVf1x2oNjuPIRuZNldpSDmiUce4QXG/qKgJcQNZ+paDDFK/G/rXHcIvV5C +du9yxxfppY7fRRMm+LFqDhKEWveG4OUhUgut71J0EREO65oMbf6UcZ2FS4XDTbFD +RSO4d8bKgf2eKwKCAQEAwAigvp8qq/yYitZeXhI6cI4ztSvwqblREhUgYWyrbUFo +a38Bey1fKQzJYUYA7raFU6alRHoHBQywjANhIaLlfvLtQfuZ4Z9CGP3Qn33uQR/E +ha4MNjjwUB0jx9lsDze1h61V95fxQLGNLVwGoaES4BpDRqvYJKnQv2X0SUigEg5U +GwryNlEW0AS/Xp/k7+PGJQernHIEWYS70FleHbAiINh+lzSfbJObgd6XnQ8IxtTr +xthXBKkkNBJdJX+/3qUQgOxTjSNUY4N9Np7myFfMvcAXuR7/K7bDegwHxffYb6Gc +v3fCFoTQFn1KTh0IvRjyv3WzqInAYVjC8CpD562VRQKCAQBirI4LnE7Q7lioMnj4 +POvO3gRZ7FSXwfZap/vEoScYMaAJeajDzVwWX6jluHmoGUVC2IahuyxMFmpy+zNl +2lcw+NKGaRuV9kYzlF62iBABgBF9aNuq7Z2TGN0dM5VkjY7AyfbJWp3D4YVt4+JS +eUlb8z1//BkK0YBZigT2RplX1l0iGn00bO/OuFYBgRPCjb9AiWWOA8rV8ZVgbSbr +M7PrqWsb4oiGw4GRUvgUMbqGCWfMoFLfvuJAmc0DaXEh9N8KbD9tuctyeg+1LalG +JDxYMjHgyCT35kisLsfA1tMei1oxIcYHaLNyVAg7Pz4TjHiDXwt0jUZWUvpQOUJ9 +kGsLAoIBAQCOqFoyAjBDIB16VpI4NDZx01IabxAUJfVSB5vMhFw9h++4m9tP1H7z +Eeqwdr7Ol40ofY4c9sIsQCcPfJs1z7vJuVIESJMih5sk0bmgIn9SpfTqkkfEKDxu +Z5djKeQa0fnrVxucGaZBtyT343uRqwVIsnn0EEk7w2OuLGFz553yi+5zQIh7TXYz +BrPb6dC7XWyfqbkVOaZ9khusRhei2mwgFnTEg3VDxcwqiF/9b2PHwfl9+M18SuL4 +RAQqjWLOVbWS8P2Ixgw0+UOVxioP/xm8hO2auqo5oUZKbpF/wgVpuJenraHj9LpZ +Wq5OpUcOo3ACR8A1nk/qgXQf0mYrwEo5AoIBAHEqA2eJVZnPiAs6U7QPAavnLxt/ +v0GLzsBBixSV8ErMToN1wfYtBb1t5fgF0Fuy85dREp1CsGJMgrnPX5bCnBmDaLl2 +Z1lUaSDcFCu+yXo+Kuy7JvSKZ4++q4ggrHvK8y8FdKH4H+56vTdXe2i9RY/v48g4 +kKyNiYtVXxrd/h47WbHF5eApheblH9hH6zC5tB/rW7Hh0nmnDcfmMW4BggbyBinH +MF3jO0YaspZOtRc2xSj8E3sGtN+f/KrBbKBb4J0j7VzuFmZC1u5grl/hx0cYE2ek +HGifmIjkKv5R4xPELoAJZyFOpN1PfS3Y+SOn0mF+RJRoGqMGcQWA3I77b5M= +-----END RSA PRIVATE KEY----- diff --git a/ctl/testdata/permissions.yaml b/ctl/testdata/permissions.yaml new file mode 100644 index 000000000..d5af09bed --- /dev/null +++ b/ctl/testdata/permissions.yaml @@ -0,0 +1,4 @@ +user-groups: + "group-id-test": + "test": "write" +admin: "group-id-test" diff --git a/holder.go b/holder.go index 88f7ff877..cfbd50801 100644 --- a/holder.go +++ b/holder.go @@ -1083,7 +1083,7 @@ func (h *Holder) LoadView(index, field, view string) (*view, error) { // CreateIndexAndBroadcast creates an index locally, then broadcasts the // creation to other nodes so they can create locally as well. An error is // returned if the index already exists. -func (h *Holder) CreateIndexAndBroadcast(cim *CreateIndexMessage) (*Index, error) { +func (h *Holder) CreateIndexAndBroadcast(ctx context.Context, cim *CreateIndexMessage) (*Index, error) { h.mu.Lock() defer h.mu.Unlock() @@ -1093,7 +1093,7 @@ func (h *Holder) CreateIndexAndBroadcast(cim *CreateIndexMessage) (*Index, error } // Create the index in etcd as the system of record. - if err := h.persistIndex(context.Background(), cim); err != nil { + if err := h.persistIndex(ctx, cim); err != nil { return nil, errors.Wrap(err, "persisting index") } diff --git a/http/client.go b/http/client.go index 183b7675d..ef21f6b24 100644 --- a/http/client.go +++ b/http/client.go @@ -142,6 +142,14 @@ func NewInternalClientFromURI(defaultURI *pnet.URI, remoteClient *http.Client, o return ic } +func AddAuthToken(ctx context.Context, req *http.Request) *http.Request { + token, ok := ctx.Value("token").(string) + if ok && token != "" { + req.Header.Set("Authorization", token) + } + return req +} + // MaxShardByIndex returns the number of shards on a server by index. func (c *InternalClient) MaxShardByIndex(ctx context.Context) (map[string]uint64, error) { span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.MaxShardByIndex") @@ -162,6 +170,7 @@ func (c *InternalClient) maxShardByIndex(ctx context.Context) (map[string]uint64 req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/json") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -194,6 +203,7 @@ func (c *InternalClient) AvailableShards(ctx context.Context, indexName string) req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/json") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -227,6 +237,7 @@ func (c *InternalClient) SchemaNode(ctx context.Context, uri *pnet.URI, views bo req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/json") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -258,6 +269,7 @@ func (c *InternalClient) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/json") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -294,6 +306,7 @@ func (c *InternalClient) IngestSchema(ctx context.Context, uri *pnet.URI, buf [] req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) if err != nil { @@ -344,6 +357,7 @@ func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, in req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { @@ -376,6 +390,7 @@ func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { @@ -404,6 +419,7 @@ func (c *InternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, indexNam } req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { @@ -434,6 +450,7 @@ func (c *InternalClient) PostSchema(ctx context.Context, uri *pnet.URI, s *pilos req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { @@ -480,6 +497,7 @@ func (c *InternalClient) CreateIndex(ctx context.Context, index string, opt pilo req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -509,6 +527,7 @@ func (c *InternalClient) FragmentNodes(ctx context.Context, index string, shard req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/json") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -540,6 +559,7 @@ func (c *InternalClient) Nodes(ctx context.Context) ([]*topology.Node, error) { req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/json") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -590,6 +610,8 @@ func (c *InternalClient) QueryNode(ctx context.Context, uri *pnet.URI, index str req.Header.Set("Authorization", token) } + req = AddAuthToken(ctx, req) + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) req.Header.Set("Content-Type", "application/x-protobuf") req.Header.Set("Accept", "application/x-protobuf") @@ -682,6 +704,7 @@ func (c *InternalClient) importNode(ctx context.Context, node *topology.Node, in req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -902,6 +925,7 @@ func (c *InternalClient) ImportRoaring(ctx context.Context, uri *pnet.URI, index httpReq.Header.Set("Accept", "application/x-protobuf") httpReq.Header.Set("X-Pilosa-Row", "roaring") httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + httpReq = AddAuthToken(ctx, httpReq) // Execute request against the host. resp, err := c.executeRequest(httpReq.WithContext(ctx)) @@ -976,6 +1000,7 @@ func (c *InternalClient) exportNodeCSV(ctx context.Context, node *topology.Node, } req.Header.Set("Accept", "text/csv") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1018,6 +1043,7 @@ func (c *InternalClient) RetrieveShardFromURI(ctx context.Context, index, field, } req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1107,6 +1133,7 @@ func (c *InternalClient) CreateFieldWithOptions(ctx context.Context, index, fiel req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1145,6 +1172,7 @@ func (c *InternalClient) FragmentBlocks(ctx context.Context, uri *pnet.URI, inde req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/json") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1194,6 +1222,7 @@ func (c *InternalClient) BlockData(ctx context.Context, uri *pnet.URI, index, fi req.Header.Set("Accept", "application/protobuf") req.Header.Set("X-Pilosa-Row", "roaring") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { @@ -1275,6 +1304,7 @@ func (c *InternalClient) TranslateKeysNode(ctx context.Context, uri *pnet.URI, i req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1329,6 +1359,7 @@ func (c *InternalClient) TranslateIDsNode(ctx context.Context, uri *pnet.URI, in req.Header.Set("Accept", "application/x-protobuf") req.Header.Set("X-Pilosa-Row", "roaring") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1360,6 +1391,7 @@ func (c *InternalClient) GetNodeUsage(ctx context.Context, uri *pnet.URI) (map[s req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1391,6 +1423,7 @@ func (c *InternalClient) GetPastQueries(ctx context.Context, uri *pnet.URI) ([]p req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request against the host. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1432,6 +1465,7 @@ func (c *InternalClient) FindIndexKeysNode(ctx context.Context, uri *pnet.URI, i req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Send the request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1480,6 +1514,7 @@ func (c *InternalClient) FindFieldKeysNode(ctx context.Context, uri *pnet.URI, i req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Send the request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1529,6 +1564,7 @@ func (c *InternalClient) CreateIndexKeysNode(ctx context.Context, uri *pnet.URI, req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Send the request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1581,6 +1617,7 @@ func (c *InternalClient) CreateFieldKeysNode(ctx context.Context, uri *pnet.URI, req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Send the request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1625,6 +1662,7 @@ func (c *InternalClient) MatchFieldKeysNode(ctx context.Context, uri *pnet.URI, req.Header.Set("Content-Length", strconv.Itoa(len(like))) req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Send the request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -1664,6 +1702,7 @@ func (c *InternalClient) Transactions(ctx context.Context) (map[string]*pilosa.T } req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx)) if err != nil { @@ -1702,6 +1741,7 @@ func (c *InternalClient) StartTransaction(ctx context.Context, id string, timeou req.Header.Set("Content-Type", "application/json") req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) if err != nil { @@ -1736,6 +1776,7 @@ func (c *InternalClient) FinishTransaction(ctx context.Context, id string) (*pil req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) if err != nil { @@ -1772,6 +1813,7 @@ func (c *InternalClient) GetTransaction(ctx context.Context, id string) (*pilosa } req.Header.Set("Accept", "application/json") req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) resp, err := c.executeRequest(req.WithContext(ctx), giveRawResponse(true)) if err != nil { @@ -1797,6 +1839,8 @@ type executeOpts struct { // giveRawResponse instructs executeRequest not to process the // respStatusCode and try to extract errors or whatever. giveRawResponse bool + // forwardAuthHeader instructs executeRequest not to follow redirects + forwardAuthHeader bool } type executeRequestOption func(*executeOpts) @@ -1806,6 +1850,11 @@ func giveRawResponse(b bool) executeRequestOption { eo.giveRawResponse = b } } +func forwardAuthHeader(b bool) executeRequestOption { + return func(eo *executeOpts) { + eo.forwardAuthHeader = b + } +} type nopCloser struct { *bytes.Reader @@ -1831,7 +1880,25 @@ func (c *InternalClient) executeRetryableRequest(req *retryablehttp.Request, opt opt(eo) } - resp, err := c.retryableClient.Do(req) + var resp *http.Response + var err error + if eo.forwardAuthHeader { + rc := retryablehttp.NewClient() + rc.HTTPClient = &http.Client{ + CheckRedirect: func(req *http.Request, via []*http.Request) error { + if len(via) > 0 { + req.Header.Set("Authorization", "Bearer "+getToken(via[0])) + } + return nil + }, + } + rc.CheckRetry = retryWith400Policy + rc.Logger = logger.NopLogger + + resp, err = rc.Do(req) + } else { + resp, err = c.retryableClient.Do(req) + } return c.handleResponse(req.Request, eo, resp, err) } @@ -1846,6 +1913,7 @@ func (c *InternalClient) handleResponse(req *http.Request, eo *executeOpts, resp if eo.giveRawResponse { return resp, nil } + if resp.StatusCode < 200 || resp.StatusCode >= 300 { defer resp.Body.Close() buf, err := ioutil.ReadAll(resp.Body) @@ -2104,6 +2172,7 @@ func (c *InternalClient) RetrieveTranslatePartitionFromURI(ctx context.Context, } req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -2171,6 +2240,11 @@ func (c *InternalClient) ImportFieldKeys(ctx context.Context, uri *pnet.URI, ind } httpReq.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + token, ok := ctx.Value("token").(string) + if ok && token != "" { + httpReq.Header.Set("Authorization", token) + } + // Execute request against the host. resp, err := c.executeRetryableRequest(httpReq.WithContext(ctx)) if err != nil { @@ -2196,6 +2270,7 @@ func (c *InternalClient) ShardReader(ctx context.Context, index string, shard ui req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/octet-stream") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -2218,6 +2293,7 @@ func (c *InternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/octet-stream") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -2227,6 +2303,30 @@ func (c *InternalClient) IDAllocDataReader(ctx context.Context) (io.ReadCloser, return resp.Body, nil } +func (c *InternalClient) IDAllocDataWriter(ctx context.Context, f io.Reader, primary *topology.Node) error { + span, ctx := tracing.StartSpanFromContext(ctx, "InternalClient.IDAllocDataWriter") + defer span.Finish() + + u := primary.URI.Path("/internal/idalloc/restore") + + // Build request. + req, err := http.NewRequest("POST", u, f) + if err != nil { + return errors.Wrap(err, "creating request") + } + + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + req.Header.Set("Accept", "application/octet-stream") + req = AddAuthToken(ctx, req) + + // Execute request. + _, err = c.executeRequest(req.WithContext(ctx)) + if err != nil { + return err + } + return err +} + // IndexTranslateDataReader returns a reader that provides a snapshot of // translation data for a partition in an index. func (c *InternalClient) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) { @@ -2244,9 +2344,10 @@ func (c *InternalClient) IndexTranslateDataReader(ctx context.Context, index str req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/octet-stream") + req = AddAuthToken(ctx, req) // Execute request. - resp, err := c.executeRequest(req.WithContext(ctx)) + resp, err := c.executeRequest(req.WithContext(ctx), forwardAuthHeader(true)) if resp != nil && resp.StatusCode == http.StatusNotFound { resp.Body.Close() return nil, pilosa.ErrTranslateStoreNotFound @@ -2273,6 +2374,7 @@ func (c *InternalClient) FieldTranslateDataReader(ctx context.Context, index, fi req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/octet-stream") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -2303,6 +2405,7 @@ func (c *InternalClient) Status(ctx context.Context) (string, error) { req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/json") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) @@ -2334,6 +2437,7 @@ func (c *InternalClient) PartitionNodes(ctx context.Context, partitionID int) ([ req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) req.Header.Set("Accept", "application/json") + req = AddAuthToken(ctx, req) // Execute request. resp, err := c.executeRequest(req.WithContext(ctx)) diff --git a/http/handler.go b/http/handler.go index 57aa77de1..2439fa539 100644 --- a/http/handler.go +++ b/http/handler.go @@ -443,8 +443,8 @@ func newRouter(handler *Handler) http.Handler { // Truly used internally by featurebase router.HandleFunc("/internal/cluster/message", handler.chkInternal(handler.handlePostClusterMessage)).Methods("POST").Name("PostClusterMessage") - router.HandleFunc("/internal/translate/data", handler.chkInternal(handler.handleGetTranslateData)).Methods("GET").Name("GetTranslateData") - router.HandleFunc("/internal/translate/data", handler.chkInternal(handler.handlePostTranslateData)).Methods("POST").Name("PostTranslateData") + router.HandleFunc("/internal/translate/data", handler.chkAuthZ(handler.handleGetTranslateData, authz.Read)).Methods("GET").Name("GetTranslateData") + router.HandleFunc("/internal/translate/data", handler.chkAuthZ(handler.handlePostTranslateData, authz.Write)).Methods("POST").Name("PostTranslateData") // other ones router.HandleFunc("/internal/fragment/block/data", handler.chkAuthN(handler.handleGetFragmentBlockData)).Methods("GET").Name("GetFragmentBlockData") @@ -566,7 +566,8 @@ func (h *Handler) chkAuthN(handler http.HandlerFunc) http.HandlerFunc { return } } - handler.ServeHTTP(w, r) + ctx := context.WithValue(r.Context(), "token", r.Header["Authorization"]) + handler.ServeHTTP(w, r.WithContext(ctx)) } } @@ -590,6 +591,7 @@ func (h *Handler) chkAuthZ(handler http.HandlerFunc, perm authz.Permission) http // put the user's groups in the context ctx := context.WithValue(r.Context(), contextKeyGroupMembership, uinfo.Groups) + ctx = context.WithValue(ctx, "token", "Bearer "+uinfo.Token) // unlikely h.permissions will be nil, but we'll check to be safe if h.permissions == nil { diff --git a/internal/authclustertests/docker-compose.yml b/internal/authclustertests/docker-compose.yml new file mode 100644 index 000000000..8590fabe4 --- /dev/null +++ b/internal/authclustertests/docker-compose.yml @@ -0,0 +1,77 @@ +version: '2' +services: + pilosa1: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + environment: + - PILOSA_NAME=pilosa1 + - PILOSA_ETCD_DIR=/root/.etcd + - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201 + - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa1:10201 + - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 + - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301 + - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301 + - PILOSA_CLUSTER_REPLICAS=3 + networks: + - pilosanet + command: + - "/featurebase server --bind pilosa1:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf" + pilosa2: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + environment: + - PILOSA_NAME=pilosa2 + - PILOSA_ETCD_DIR=/root/.etcd + - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201 + - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa2:10201 + - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 + - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301 + - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301 + - PILOSA_CLUSTER_REPLICAS=3 + networks: + - pilosanet + command: + - "/featurebase server --bind pilosa2:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf" + pilosa3: + build: + context: ../.. + dockerfile: Dockerfile-clustertests + image: ptest + environment: + - PILOSA_NAME=pilosa3 + - PILOSA_ETCD_DIR=/root/.etcd + - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS=http://0.0.0.0:10201 + - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS=http://pilosa3:10201 + - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 + - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301 + - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301 + - PILOSA_CLUSTER_REPLICAS=3 + networks: + - pilosanet + command: + - "/featurebase server --bind pilosa3:10101 -c /go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/featurebase.conf" + client1: + build: + context: . + dockerfile: ../clustertests/Dockerfile + depends_on: + - "pilosa1" + - "pilosa2" + - "pilosa3" + environment: + - ENABLE_PILOSA_CLUSTER_TESTS=1 + - GO111MODULE=on + - PROJECT=authclustertests + - ENABLE_AUTH=1 + networks: + - pilosanet + volumes: + - /var/run/docker.sock:/var/run/docker.sock + command: + - "cd /go/src/github.com/molecula/featurebase/ && go test -mod=vendor -v -count=1 github.com/molecula/featurebase/v3/internal/clustertests" +networks: + pilosanet: diff --git a/internal/authclustertests/testdata/certs/README.md b/internal/authclustertests/testdata/certs/README.md new file mode 100644 index 000000000..4c1009a9c --- /dev/null +++ b/internal/authclustertests/testdata/certs/README.md @@ -0,0 +1,12 @@ + + +# these test certs were generated with the following commands + +certstrap --depot-path certs init --common-name pilosa-ca --expires "100 years" +certstrap --depot-path certs request-cert --common-name localhost --domain localhost +certstrap --depot-path certs sign "localhost" --CA pilosa-ca --expires "100 years" + +# certstrap version +dev-25ea708a + +(built with go 1.13) diff --git a/internal/authclustertests/testdata/certs/localhost.crt b/internal/authclustertests/testdata/certs/localhost.crt new file mode 100644 index 000000000..8269ccda6 --- /dev/null +++ b/internal/authclustertests/testdata/certs/localhost.crt @@ -0,0 +1,25 @@ +-----BEGIN CERTIFICATE----- +MIIEPjCCAiagAwIBAgIRAJ7rl74WPv8pLuhVRXt6fV0wDQYJKoZIhvcNAQELBQAw +FDESMBAGA1UEAxMJcGlsb3NhLWNhMCAXDTIwMTAyMDE5MTMzNFoYDzIxMjAxMDIw +MTkxMzE5WjAUMRIwEAYDVQQDEwlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUA +A4IBDwAwggEKAoIBAQDmi8FMWt23M0Cr2aCgEXUGQ0gv/4M7CXH/5GkSI866YwGV +Bd1iZMBRiONQwvGDnqYZRrAQv6mFjfyBqxdkbh++74FC3JK7sLhks0vg5VwbHV7T +5kj3bJqd+LKn5qPPOQXX9sgmv/NkggF/XXwF73noLPmgDQ78S+OP0ANmi1TQiU3a +gE+qp+Qpl5KC7dH9aC9nvE9iGfEcGNr+rXj05liiXqe4ZtIKWjeke7Ej64C6qX97 +bNPzmLARtqbRsIkfAU8SJy3YuHfW8n1xr4B7ENm9jHQCh1wUv2YhaPpnio+/R2zp +Lw4yCqilDX9ZZ4nG3cBFuziSf+BUXJ9ydbw1aX8HAgMBAAGjgYgwgYUwDgYDVR0P +AQH/BAQDAgO4MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAdBgNVHQ4E +FgQUJRQJpaR5bp4ZyUsMxgK+UJl6PSgwHwYDVR0jBBgwFoAU69lmXSa5BZeyYU/6 +XpWdtr59H1YwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IC +AQChxsBZ/b14ukJXX48BxAyZcy5r7GrLcRGQ3guUTONFVDWPzzpd8mjHi0yJDhMW +2zWtw3/H+c+zT7uRd+2sUxFdpAurNSFCdV++5Q/0aFvl+By5+MhVhtznEQDU0/lM +zFxiEYe/N9Vi2N0S1KPxvYL/RfBU27u+O/50zhjueM1BTyHTTqL6E2DFeT2VPKIg +zCDUtiTEDFZrD0XGITT/3CIoNCK8aC+Fq65OEoyEn6qR5qg1Kc4tfZmo6hWYiSlR +XeP36cP9R8kEMte1BdE74GVqE9cTuVZERdgB0hv3EME7Byq7uIm/a+JXbsh2/OFm +HcE0/HP+O0YK8YaVMGwI3pZYy2syWqPcakcvusETehr6P+Ihh2cOKRwqkCl6b87e +uSLJNTUMKZgakW6Bjv6lgQaWqnKzTC/RgmQ+G3w0nKATX9+jYE2j3MzZhbtcml+2 +gp6u225yAJaYt/MQidwUMiKYeCgjaUNoL0fOJesGkokPk80ceISnqvbSRiZRTvK1 +bVenkhkBrHuvvgKVstzcuZI9oQ2snWhK1naVQiOtQNEFUCHwyU95zADOK0km88NB +2het6yYaEUL9csHPEjPd3lFglerGQnil2Ly1slUC4jb7hfVRHjOFs8PVr9gQ45dW +Jvsv4pawHKFE0ennoNvoDmzbiY1TY5ScTZquPGIsEBV+tQ== +-----END CERTIFICATE----- diff --git a/internal/authclustertests/testdata/certs/localhost.csr b/internal/authclustertests/testdata/certs/localhost.csr new file mode 100644 index 000000000..1814b72af --- /dev/null +++ b/internal/authclustertests/testdata/certs/localhost.csr @@ -0,0 +1,16 @@ +-----BEGIN CERTIFICATE REQUEST----- +MIICgDCCAWgCAQAwFDESMBAGA1UEAxMJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0B +AQEFAAOCAQ8AMIIBCgKCAQEA5ovBTFrdtzNAq9mgoBF1BkNIL/+DOwlx/+RpEiPO +umMBlQXdYmTAUYjjUMLxg56mGUawEL+phY38gasXZG4fvu+BQtySu7C4ZLNL4OVc +Gx1e0+ZI92yanfiyp+ajzzkF1/bIJr/zZIIBf118Be956Cz5oA0O/Evjj9ADZotU +0IlN2oBPqqfkKZeSgu3R/WgvZ7xPYhnxHBja/q149OZYol6nuGbSClo3pHuxI+uA +uql/e2zT85iwEbam0bCJHwFPEict2Lh31vJ9ca+AexDZvYx0AodcFL9mIWj6Z4qP +v0ds6S8OMgqopQ1/WWeJxt3ARbs4kn/gVFyfcnW8NWl/BwIDAQABoCcwJQYJKoZI +hvcNAQkOMRgwFjAUBgNVHREEDTALgglsb2NhbGhvc3QwDQYJKoZIhvcNAQELBQAD +ggEBABMi2/4j1/qzwWAYlEs2KW3z+apzzDLKgjE0kY6QvELh/8aBj0rMglb0HM2x +4iSSoX1ZwZgDZ9fIJ3klG/UF7CUweMghb9yC2PP9Z8WuqaECQyM87KgSln8PND9E +1OvD30rp9yr9KxEeckq+c1ebLi/qGrIY21VCwfxA0mv3sfi7Q5ONIckay/Xj+1Tz +ovE/TkM/8wTE/SKbpQSCkP7K1NDXuAhMGjcN0x3d3f8nBcLcZOrRroiHy38Bv/9T +Vd62IY6uqYw9sluBbMX72D/mmJiCKEw3+DhDJFhHCTCrAQM0QwLuwnG2lQFYoENc +ZAkwDIi+3DXHEEyloNSYGtXMEiA= +-----END CERTIFICATE REQUEST----- diff --git a/internal/authclustertests/testdata/certs/localhost.key b/internal/authclustertests/testdata/certs/localhost.key new file mode 100644 index 000000000..b7434fdc9 --- /dev/null +++ b/internal/authclustertests/testdata/certs/localhost.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA5ovBTFrdtzNAq9mgoBF1BkNIL/+DOwlx/+RpEiPOumMBlQXd +YmTAUYjjUMLxg56mGUawEL+phY38gasXZG4fvu+BQtySu7C4ZLNL4OVcGx1e0+ZI +92yanfiyp+ajzzkF1/bIJr/zZIIBf118Be956Cz5oA0O/Evjj9ADZotU0IlN2oBP +qqfkKZeSgu3R/WgvZ7xPYhnxHBja/q149OZYol6nuGbSClo3pHuxI+uAuql/e2zT +85iwEbam0bCJHwFPEict2Lh31vJ9ca+AexDZvYx0AodcFL9mIWj6Z4qPv0ds6S8O +MgqopQ1/WWeJxt3ARbs4kn/gVFyfcnW8NWl/BwIDAQABAoIBAFX+GPqfBgY4cs3m +3ff2qvzMCdgFaXCS5Fe7XcmrW4fAOC3awynZRLbk5U0Reb5LZc8Vw8RriRLM1DuV +kqMeRG8WrNNArOafUxgUnJ/lTUa73MwTIHJRqxZzVkg0SjOYJGranOt/O4zoxSA5 +wXIBUipc5Dtjw4wtzlKtFyefnuItL2MCdwOHUdZfnhr9Oykp1fuNqBqkkeryj3XV +ukHQvqU5zkMSayprNglziqTHUzU33iyZeDng+CJQeYTEc7Gn+zja2SFFBlPHqXXo +/OzAr94zI3vOnj3yRM3+sKMJVPV+RoJEGpsvPVuVn38d1VnIMEx8Gy/wif6tmM9c +7Q44hKECgYEA/JMFwkPGbry80ktDI065k5FIYn1EDRyUaQqyskmkBRcNW3qOShqj +o/zWQfCgxP587IEdKBBwqCpdqfghi3EW+JqfVlbGY6t1chAurYF/47CTIgKO5qRM +GdCY2OdiAeo5nba/KiLQfSuY08MCNDrQabLRJXIng8qVWpRwQzsv5rECgYEA6aw/ +HugeQhTxk2uV91jJaAQIaxrt6JxuoG0CGGlbDrrTl2dnbPYA0muHMFdT/bzKjCpv +n/ScqbCyHuy+lWSnOzgedRNQCB46+0H58LAjITAj9QaT3raZqVReVIaD+pnx27dp +Cw5Ws6ENa9AQey3DO+dkRWot2AcLSw6TGR8HnzcCgYEA/ArfCU/G2cSgDJ6sPbSW +vaqR+C6W1Rq7AuN5FS8lbSrm2m2/RjW1LLTnPmAYntxx3zSs2sklErtMQovpNZRB +3w21iVwIl3eHOK7rVZtP+u++s4aoAYLcqjod/P1RMSYCHt85fpvFP9Ncq50DOwmh +5ohZ6ysyQXLMfdp4+K48i9ECgYEApOFALKu2ZgRnLRFV4REKFFX8Jq76vg5bVOF2 +AAmfEbasBIIXDWBL1i2/V1HXVwv2k46B8wjj3ixqkr2UAM/j3DpN62g0KXZDQfUc +ykNOlmVkickZX6XSqRN5+ARubc5gRRuWiBGXBeqXEMLgTjpNLyCntP8l1++ofU6M +ZsZpV2MCgYA3nfNXAR5O4B/dm/2HmDQrXy0qia7Hwi/95pgL2FJaEmjBCPI1j12o +M5YCbhpr1pwsNKPV9AUlUz+OCwS8Vt+V0gQf9/XvNOsifU+mbMYVpuNGwmcKafnv +qECSeidrmhWJSR/SSNBcE94im/8ObVU110WJMkjC9otjDl9Aua/3LQ== +-----END RSA PRIVATE KEY----- diff --git a/internal/authclustertests/testdata/certs/pilosa-ca.crl b/internal/authclustertests/testdata/certs/pilosa-ca.crl new file mode 100644 index 000000000..3b25dd052 --- /dev/null +++ b/internal/authclustertests/testdata/certs/pilosa-ca.crl @@ -0,0 +1,16 @@ +-----BEGIN X509 CRL----- +MIIChTBvAgEBMA0GCSqGSIb3DQEBCwUAMBQxEjAQBgNVBAMTCXBpbG9zYS1jYRcN +MjAxMDIwMTkxMzIyWhgPMjEyMDEwMjAxOTEzMjBaMACgIzAhMB8GA1UdIwQYMBaA +FOvZZl0muQWXsmFP+l6Vnba+fR9WMA0GCSqGSIb3DQEBCwUAA4ICAQBja+EDQAp+ +KeD7UhWMMrTd9j03GgQ2E2Z7+Ba0qJ5+kS7/t+Yja2o5dQJkrC3GwEMOQb6DRRUE +nUE4xlr5Rryoq0dZk+Lp1f4cHrnP8l1xylUL44gsnY4v8zMR8L8X98vj7kKCqB8w +DFX7qkMlE5Ie2Hha7uuOJ85FnxIbMcRxFQH2m2zDfWG8/Lmxezvv9Hn45V/kwIQy +MmBh6cNuhzEneyNpM9yMRe/29QgVitF/2q6d+FzK8w8hkUFeYlyM+cP7F4Ml7160 +UidSQM04zvBtJ8frZAvrDaPBBZhrTXcyw6+Qnp/aaW1ZsEIdHEcbYGNdgtazleoG +VH35cDP90KfiRbq69PQ9Zqn3cI//MX3sHrglA9wsEhHc9P7dowHaOFyxPouZPEmQ +/Jqg5oyJzujRwhf0v3SdJvhuDEzla2N+QyYRk0kRHtdv+glz7T7CnTYCk+DTv+oh +QABUrCbjfBoE5M2Qep9ZkIbl2gaDCpvbZSF4zFLKQc2aIOBpVn3HgTGBvdFD3FJY +Txl2F4Y3rS1T/WMAH86cZIc9h5HlMdFtAFnHAlHtB3wGw3FD/GcvGcvz2D4GaxKq +erzrnOxjYOA4M0haGzWF6dC7aPA8y35eZuqNXvbenTtc7A11bWTJfG1I7ctvLyPE +MpCNMHfymh/XtYZiZhvu6ueu3OeKScN+tA== +-----END X509 CRL----- diff --git a/internal/authclustertests/testdata/certs/pilosa-ca.crt b/internal/authclustertests/testdata/certs/pilosa-ca.crt new file mode 100644 index 000000000..9878e3aa7 --- /dev/null +++ b/internal/authclustertests/testdata/certs/pilosa-ca.crt @@ -0,0 +1,29 @@ +-----BEGIN CERTIFICATE----- +MIIE6jCCAtKgAwIBAgIBATANBgkqhkiG9w0BAQsFADAUMRIwEAYDVQQDEwlwaWxv +c2EtY2EwIBcNMjAxMDIwMTkxMzIyWhgPMjEyMDEwMjAxOTEzMjBaMBQxEjAQBgNV +BAMTCXBpbG9zYS1jYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBALT/ +uNmbnfXWNX+FsL0Waqw/5deti5F4cSjMrGRQpXxalTcooqNk/lkeqXkvi9ooFROZ +/HyQR9GM9dSD/aj6gD3FnGA4ueB24Xr6bWsRpDRh6+3UGLB3YCNNdGLSfX3LPMYh +RJutFmsg+r6SrSytbLbffu+0a/4fxtajZNwQJjDjd8qflXQZYlzp2LHk1A/jqqdI +fBtqkNg925TGKiavvUqKtdI/eFzRoiQ7NLBUJmszzveUXvOUMsMnW2/myLBe3Oqk +Vsy85lya0ADln20C3Lb0+ZA4KoGX3EWdtBXEuWqMoyvCJoJ4I3bH2LlfOUjRt8UE +pPk6sPMROJ+75mlgvgnSlYsN8PaZdvdm2VGVWRWUyEfyW/qa2fv8d2XBWqibl0YF +tqay9CX1aWgC9q12yx3vj7Yh+ZNbeZFLc7IL8zyNMwIjIOIIyGBY70KewfgVktzq +fAMz6h1sr9Kxozil97Cu3ma4B6UiL3rUbYMO/rNhVxcIuUoJpgIEVuRt+uXEG74y +XftauZ67qILFQzfpoacncvDEx5nJ3itLgbbyt1n1iWdGuEiMSLFT+x+nNUgVpQgA +sWRYxHdisM4xzRVN6pAaToMs1p8Ju7l9xU3z7RSogTyVk9gMIIV4t9TDYP5fI10Y +GFi7B6q0t3pIGXgKySHjCSl0EKYkDQDFl5tWeaiXAgMBAAGjRTBDMA4GA1UdDwEB +/wQEAwIBBjASBgNVHRMBAf8ECDAGAQH/AgEAMB0GA1UdDgQWBBTr2WZdJrkFl7Jh +T/pelZ22vn0fVjANBgkqhkiG9w0BAQsFAAOCAgEAnlBFrWhB+WesCc3lhK980rA6 +roNFYMZdaXvg4zaEGergkRvPab5yXoof1AAeznJm45GQfXn8HbQlrZmAqWg3fNld +/TX+jNvosM8K8K+PzesDGHsm/eQnbrb0qzMDsQgFY+nnD+x/ZQtjmKZtcNr/0ZlM +EJeXWU5cGy70GMbNztspMHsOLa3ZDLsBOJYOwSFxDlLDFrjZoRoPCWw8jRL+Tb4t +JjZcGZDD4a5+DqcojanIdNU1yI4teP6aV1LQTVNn4pwOap+tD0De/WzOPmXTQq5M +9ssxL7xSqVShQQMC8LVSWSRxtT6kLq0Av6i7wio0DZGnH3ynERTUs13DRZkwbVsE +OaQLmiQnsHRTIpdts/fswZ2FRPvdhhXxBjiGQZGEGXXznxHTNJ6nQioh95Ft5hNA +82i8Z74miaFIT/33/sZ5SuwUzphCgqCY2x7NUS8J313O9lsar0bweJTvQaZg/69E +PmmwUcDebh+pgKP01z4BqTzhtchmFUKzT+oOC8tmTeSlhsBTzO4xMw6OqhpXKL+c +k9f2CGUZYtEZHDRmP+C++FEi+B/tV2Oq3on+QPiaIIcRsOftthGUvJ8htUl3w+hq +B5TnL8CeLjXGKKRp+UiakrB4E7y2aIbrtIRnJ/Llg2XMND/0xbldsNsyDNXCIoDH +sz8HqwF3CUbv5XD4ioY= +-----END CERTIFICATE----- diff --git a/internal/authclustertests/testdata/certs/pilosa-ca.key b/internal/authclustertests/testdata/certs/pilosa-ca.key new file mode 100644 index 000000000..135a6e233 --- /dev/null +++ b/internal/authclustertests/testdata/certs/pilosa-ca.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKAIBAAKCAgEAtP+42Zud9dY1f4WwvRZqrD/l162LkXhxKMysZFClfFqVNyii +o2T+WR6peS+L2igVE5n8fJBH0Yz11IP9qPqAPcWcYDi54HbhevptaxGkNGHr7dQY +sHdgI010YtJ9fcs8xiFEm60WayD6vpKtLK1stt9+77Rr/h/G1qNk3BAmMON3yp+V +dBliXOnYseTUD+Oqp0h8G2qQ2D3blMYqJq+9Soq10j94XNGiJDs0sFQmazPO95Re +85Qywydbb+bIsF7c6qRWzLzmXJrQAOWfbQLctvT5kDgqgZfcRZ20FcS5aoyjK8Im +gngjdsfYuV85SNG3xQSk+Tqw8xE4n7vmaWC+CdKViw3w9pl292bZUZVZFZTIR/Jb ++prZ+/x3ZcFaqJuXRgW2prL0JfVpaAL2rXbLHe+PtiH5k1t5kUtzsgvzPI0zAiMg +4gjIYFjvQp7B+BWS3Op8AzPqHWyv0rGjOKX3sK7eZrgHpSIvetRtgw7+s2FXFwi5 +SgmmAgRW5G365cQbvjJd+1q5nruogsVDN+mhpydy8MTHmcneK0uBtvK3WfWJZ0a4 +SIxIsVP7H6c1SBWlCACxZFjEd2KwzjHNFU3qkBpOgyzWnwm7uX3FTfPtFKiBPJWT +2AwghXi31MNg/l8jXRgYWLsHqrS3ekgZeArJIeMJKXQQpiQNAMWXm1Z5qJcCAwEA +AQKCAgAWmjiDNCOtp2pW2mMPudToXbJeFJXxPJEk/yon/MotlUI8+R4WOW5pwqJ3 +N7DHNWosYHZfN8VALdIlD7aFe4K4NA0rFupfVXki2lL/o9xVjkTgFjRfFQk0X1/B +V3fEVbTpKQ5gQmUiS6QEWFy3z5Bb5dz8IhO6UE2MUCswL/QU9tLmwrbvIJxf7fPZ +gzHYKh4NdcfJxK0B0/evxG9PFXMV8+xwrOxi6urMi3gw7NE/YeDemfChikgshqWs +e61kGPSNeKg+OPirZ8nB0urtugXF8yGXGOx18njXWLI8Zayh2Z4mwL/+WvJSyvIN +dA67QTUprULMvL+MGwJvMA+96Q7SBRVKR9HHNaP9pFsZup3QX9mqDQ9miB6+rzn7 +f5RiSLVgq+HUPMPfqgXCkQZBcY28TcM1BZhS4uJJTkbvVTkrlHKJs6Q/LBqJFvq0 +3+2M1xQb4HdRTRlwZ/YsxdqXGIoA3Xx3nZbb6LlPp/MT93xxsLJNNP47n445Cw8i +lz7hJJDwo+TyXmRRWKlFXO8TEhqKhK9ZEXmkXBxSeCQV0oTYS5kU9XdZ5iu5/CQQ +Lv+uFQfHTPWm/Lp5RC8JEEJwK5bwRAs5d9oWg5EbWJp2ol36g3YO1np5RFsQn3jL +qJPz35X3Bp9zQeZcAZpt1fWFdyX9f7V2LLCY8gUyIlfCEXBWwQKCAQEA8Uoes4ph +15tM1LswmOLqOq9iWuvcKYSNcz8my8nlP5zkUdGKvGcLblm5Mg4wKoPFiyjKhX7I +S8DUN8x7E5aiNBiZ0PGku4CKjubQgdFG/rYRfnrUEaA81Uw3a69eYioKbxhhxDzb +Lqd0/tGNHQZBQEyofOChwqTWpAdIh79F1oXejcPzRDr+oXxJVnvJDdYCEN/0TYkz +qeJdEtnVf1x2oNjuPIRuZNldpSDmiUce4QXG/qKgJcQNZ+paDDFK/G/rXHcIvV5C +du9yxxfppY7fRRMm+LFqDhKEWveG4OUhUgut71J0EREO65oMbf6UcZ2FS4XDTbFD +RSO4d8bKgf2eKwKCAQEAwAigvp8qq/yYitZeXhI6cI4ztSvwqblREhUgYWyrbUFo +a38Bey1fKQzJYUYA7raFU6alRHoHBQywjANhIaLlfvLtQfuZ4Z9CGP3Qn33uQR/E +ha4MNjjwUB0jx9lsDze1h61V95fxQLGNLVwGoaES4BpDRqvYJKnQv2X0SUigEg5U +GwryNlEW0AS/Xp/k7+PGJQernHIEWYS70FleHbAiINh+lzSfbJObgd6XnQ8IxtTr +xthXBKkkNBJdJX+/3qUQgOxTjSNUY4N9Np7myFfMvcAXuR7/K7bDegwHxffYb6Gc +v3fCFoTQFn1KTh0IvRjyv3WzqInAYVjC8CpD562VRQKCAQBirI4LnE7Q7lioMnj4 +POvO3gRZ7FSXwfZap/vEoScYMaAJeajDzVwWX6jluHmoGUVC2IahuyxMFmpy+zNl +2lcw+NKGaRuV9kYzlF62iBABgBF9aNuq7Z2TGN0dM5VkjY7AyfbJWp3D4YVt4+JS +eUlb8z1//BkK0YBZigT2RplX1l0iGn00bO/OuFYBgRPCjb9AiWWOA8rV8ZVgbSbr +M7PrqWsb4oiGw4GRUvgUMbqGCWfMoFLfvuJAmc0DaXEh9N8KbD9tuctyeg+1LalG +JDxYMjHgyCT35kisLsfA1tMei1oxIcYHaLNyVAg7Pz4TjHiDXwt0jUZWUvpQOUJ9 +kGsLAoIBAQCOqFoyAjBDIB16VpI4NDZx01IabxAUJfVSB5vMhFw9h++4m9tP1H7z +Eeqwdr7Ol40ofY4c9sIsQCcPfJs1z7vJuVIESJMih5sk0bmgIn9SpfTqkkfEKDxu +Z5djKeQa0fnrVxucGaZBtyT343uRqwVIsnn0EEk7w2OuLGFz553yi+5zQIh7TXYz +BrPb6dC7XWyfqbkVOaZ9khusRhei2mwgFnTEg3VDxcwqiF/9b2PHwfl9+M18SuL4 +RAQqjWLOVbWS8P2Ixgw0+UOVxioP/xm8hO2auqo5oUZKbpF/wgVpuJenraHj9LpZ +Wq5OpUcOo3ACR8A1nk/qgXQf0mYrwEo5AoIBAHEqA2eJVZnPiAs6U7QPAavnLxt/ +v0GLzsBBixSV8ErMToN1wfYtBb1t5fgF0Fuy85dREp1CsGJMgrnPX5bCnBmDaLl2 +Z1lUaSDcFCu+yXo+Kuy7JvSKZ4++q4ggrHvK8y8FdKH4H+56vTdXe2i9RY/v48g4 +kKyNiYtVXxrd/h47WbHF5eApheblH9hH6zC5tB/rW7Hh0nmnDcfmMW4BggbyBinH +MF3jO0YaspZOtRc2xSj8E3sGtN+f/KrBbKBb4J0j7VzuFmZC1u5grl/hx0cYE2ek +HGifmIjkKv5R4xPELoAJZyFOpN1PfS3Y+SOn0mF+RJRoGqMGcQWA3I77b5M= +-----END RSA PRIVATE KEY----- diff --git a/internal/authclustertests/testdata/featurebase.conf b/internal/authclustertests/testdata/featurebase.conf new file mode 100644 index 000000000..8661df8cf --- /dev/null +++ b/internal/authclustertests/testdata/featurebase.conf @@ -0,0 +1,383 @@ +# FEATUREBASE HOST CONFIGURATION +# +# Uncomment when/where appropriate + +# ============================================================================== +# Use advertise to specify the address advertised by the server to other nodes +# in the cluster and to clients via /status endpoint. Host defaults to IP +# address represented by bind parameter with network port. +# +# advertise = :10101 +# advertise-grpc = :20101 + + + +# "long-query-time" represents duration of time that will trigger log and stat +# message for queries longer than X time. Ex. "1m30s" 1 minute 30 seconds +# +# long-query-time = "10s" + + + +# Unique name for node in cluster. This is just a human-readable label for +# convenience and not used by any underlying logic. + +# name = "pilosa1" + + + +# # Host:Port where Featurebase server listens for HTTP requests. +# # Default is localhost:10101 +# # +# bind = "pilosa1:10101" + + + +# # The address and port featurebase will listen to for all GRPC connections +# # Ex. python-molecula, grafana for queries, etc. +# # +# bind-grpc = "localhost:20101" + + + +# Directory to store Featurebase data files +# data-dir = "/var/lib/molecula" + + +# ============================================================================== +# CORS (Cross-Origin Resource Sharing) Allowed Origins +# List of allowed origin URIs for CORS +# +# [handler] +# allowed-origins = ["https://myapp.com", "https://myapp.org"] + + + +# Path to the log file +# log-path = "/var/log/molecula/featurebase.log" + + + +# Verbose - Enable verbose logging. Valid options are true or false. +# Set to true only when debugging as directed by Molecula engineers. +# +# verbose = true + + + +# Soft limit on max number of files featurebase will keep open simultaneously. +# When past this limit, featurebase will only keep files open for as long as is +# needed to write updates. +# +# max-file-count = 900000 + + + +# Maximum number of active memory maps featurebase will use for fragment files. +# Actual total usage may be slightly higher. +# Best practice is to set this to ~10% lower than your system's max map count. +# See sysctl vm.max_map_count in Linux. +# +# max-map-count = 900000 + + + +# Max Writes Per Request - Max number of mutating commands allowed per request. +# This includes Set, Clear, ClearRow, and Store +# +# max-writes-per-request = 5000 + + + +# The following option sets the maximum number of queries that are maintained +# for the /query-history endpoint. +# This parameter is per-node, and the result combines the history from all nodes. +# +# query-history-length = 100 + + + +# External database to connect to for `ExternalLookup` queries. +# lookup-db-dsn = "postgres://localhost:5432/db" + + + +# ============================================================================== +# For cluster stanza, "name" represents name for cluster. Must be same on all +# nodes in cluster. "replicas" represents number of hosts each piece of data +# should be stored on. Must be greater than or equal to 1 & less than or equal +# to number of nodes in cluster. +# [cluster] +# name = "cluster1" +# replicas = 1 + + + +# ============================================================================== +# [etcd] +# etcd is the tool Featurebase uses for node-to-node, intra-cluster +# communication. etcd is embedded in the featurebase cluster rather than +# running as a separate instance. +# It's important to configure this correctly for your network and nodes, and +# that it is consistent across all nodes. +# +# The easiest setup can be used when all nodes can reach all other nodes via a +# local subnet: +# listen-peer-address = advertise-peer-address +# = (what's in the initial-cluster-list) +# = the nodes ip address (which can be reached by every +# other node +# (localhost:10401 would not work for this, as each node can't reach that) +# +# If each node is separated by a proxy, or must be reached via url / dns, you +# will need to use a more complicated setup: +# listen-peer-address = the nodes local ip address +# (specific ip, localhost, or 0.0.0.0 for all +# local ip's) +# advertise-peer-address = the nodes ip address, reachable by all other nodes +# (This address should also be included in +# initital-cluster-list) +# in this case, you specify a different url/ip for listen-peer and +# advertise-peer. E.g. you specify 0.0.0.0 for listen, or (like in their case) +# you use a url for advertise. In each of these cases, you should set listen +# to the local ip, and you set advertise = to how each other node connects to +# this node, and you also use this same address in the initial cluster. +# The key here is that initial-cluster has to include the same node name and +# advertise-peer address as the node it's on (edited) + + + +# for additional assistance, and for help with config issues, +# see https://etcd.io/docs/v3.5/faq/ cluster-url - URL of existing cluster +# that a new node should join when adding nodes to cluster. +# +# cluster-url = "http://localhost:10401" + + + +# Address and port to bind to for client communication +# listen-client-address = "http://localhost:10401" + + + +# Address and port to bind to for peer communication +# listen-peer-address = "http://localhost:10301" + + + +# Comma-separated list of node=address pairs that makes up initial cluster when +# first started. In each pair, "node" value (left side of = ) should match +# name of node specified by "name" configuration parameter +# +# initial-cluster = "featurebase1=http://localhost:10301" + + + +# ============================================================================== +# Profile Block Rate - Block Rate is passed directly to Go's +# runtime.SetBlockProfileRate. Goroutine blocking events will be sampled at 1 +# per rate nanoseconds. A value of "1" samples every event, and 0 disables +# profiling. +# +# block-rate = 10000000 + +# Profile Mutex Fraction - Mutex Fraction is passed directly to Go's +# runtime.SetMutexProfileFraction. 1/ fraction of events will be sampled. +# +# mutex-fraction = 100 + + + +# ============================================================================== +# PostgreSQL Section +# [postgres] +# +# Endpoint Bind - Address to bind a PostgreSQL wire protocol endpoint. +# No PostgreSQL endpoint will be exposed unless a bind address is specified. +# Requires Molecula v3.0 or newer. +# +# bind = "localhost:55432" + + + +# The PostgreSQL endpoint has support for a connection limit. +# This is generally not necessary, so it is disabled by default. +# +# connection-limit = 10000 + + + +# PostgreSQL Max Startup Packet Size - By default, the postgres endpoint +# uses an 8 MiB limit on incoming PostgreSQL startup packets. This should +# typically be sufficient, but may be exceeded if a client sends an unusually +# large amount of configuration data. Oversized startup packets are typically +# caused by connecting with a different protocol, e.g. HTTP. +# +# max-startup-size = 10000000 + + + +# PostgreSQL Timeouts +# In order to detect stalled clients, the PostgreSQL endpoint has connection +# read and write timeouts. There is also a startup timeout, which is used for +# connection setup. The read timeout does not impact idle connections. Idle +# connections will only be closed by the server if TCP keepalive reports a +# break in the connection. TCP keepalives use the default configuration +# provided by the host. +# Caution: Due to a limitation of the PostgreSQL wire protocol, +# raising the write timeout may delay the shutdown of a featurebase node. +# +# startup-timeout = "20s" +# read-timeout = "20s" +# write-timeout - "20s" + + + +# Postgres Endpoint TLS - TLS configuration for the PostgreSQL endpoint is +# structured the same as the TLS configuration for Featurebase's other endpoints, +# but placed under [postgres.tls]. If TLS is configured on the postgres endpoint, +# Featurebase will reject unsecured connections. +# [postgres.tls] +# certificate = "/Users/souhailanoor/tls/out/auth.mybusiness.com.crt" +# key = "/Users/souhailanoor/tls/out/auth.mybusiness.com.key" +# ca-certificate = "/Users/souhailanoor/tls/out/auth.mybusiness.com.crt" +# enable-client-verification = true + + + +# ============================================================================== +# Usage Duty Cycle - Featurebase maintains a disk/memory usage cache that is +# calculated periodically in the background and accessed by the UI/usage +# endpoint. Since this disk scan can take a long and unpredictable amount of +# time, its timing behavior is specified in a relative, rather than absolute +# sense. That is, the duty cycle sets the percentage of time that is spent +# recalculating this cache. This setting affects the results received from +# the "/ui/usage" http endpoint, as well as all data file and memory usage +# values and graphs on the webui "tables" page + +# Special considerations: +# * If disk usage can be calculated quickly (less than 5 seconds), fresh +# results will be calculated when accessed +# * When disk usage takes longer to calculate, there is a minimum of one +# hour wait between cache recalculations +# Setting this value to 0 will completely disable the calculation of disk usage +# +# usage-duty-cycle = 20 + + + +# ============================================================================== +# Use [metric] stanza to define attributes for monitoring. +# [metric] +# Specify which service to use for collecting metrics. Valid options are: +# "statsd", "expvar", "prometheus", "none" +# +# service = "prometheus" + + + +# Remote host to send statsd metrics to. +# host = "localhost:8125" + + + +# The interval to send statsd metrics. +# poll-interval = "10s" + + + +# Debugging flag to enable to send diagnostic information to Featurebase +# developers. +# +# diagnostics = false + + + +# ============================================================================== +# TLS Certificate Section - Path to TLC certificate used for service HTTPS. +# Suffix should contain .crt or .pem + +[tls] + certificate = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/certs/localhost.crt" + key = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/certs/localhost.key" + +# ============================================================================== +# Tracing Section +# [tracing] +# +# Jaeger sampler type. Valid options are: "const, "probabilistic", "ratelimiting", +# or "remote". Set to 'off' to disable tracing completely. +# +# sampler-type = "remote" + + +# Jaeger sampler parameter (number) +# sampler-param = 0.001 + + + +# Tracing Agent Host:Port +# agent-host-port = "localhost:6831" + + + +# ============================================================================== +# Configuration for the RBF storage format. +# [rbf] +# Maximum size for each RBF database file. +# Allocates virtual memory but does not preallocate physical disk space. +# If you get into the range where you have 16000 shards on a single node +# (across all indexes), you will need to lower this in order to not run out of +# virtual address space. +# +# max-db-size = 4294967296 + + + +# Maximum size for each RBF WAL file. +# Allocates virtual memory but does not preallocate physical disk space. +# This is the same as max-db-size, but for the write-ahead log. If you set it +# smaller, set max-wal-checkpoint-size to 1/2 of this (we will likely +# condense these options in the future). +# +# max-wal-size = 4294967296 + + + +# Minimum WAL size before WAL pages can be copied to the main database file. +# min-wal-checkpoint-size = 1048576 + + + +# Maximum WAL size before transactions are halted to copy WAL pages to the +# main database file. +# +# max-wal-checkpoint-size = 2147483648 + + +# ============================================================================== +# [storage] +# Sync all changes to the file system. +# Should not be changed in production systems unless you know what you are +# doing - Should always be on unless testing or possibly while performing a +# bulk import and you are not worried about data loss +# +# fsync = true + + +# ============================================================================== +# Enable/Disable AuthN/AuthZ for featurebase +# Can choose identity provider, pass authorize and user-info endpoints, and client id +[auth] + enable = true + client-id = "e9088663-eb08-41d7-8f65-efb5f54bbb71" + client-secret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" + authorize-url="https://login.microsoftonline.com/organizations/oauth2/v2.0/authorize" + token-url="https://login.microsoftonline.com/organizations/oauth2/v2.0/token" + group-endpoint-url = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true" + logout-url = "https://login.microsoftonline.com/common/oauth2/v2.0/logout" + scopes = ["https://graph.microsoft.com/.default", "offline_access"] + secret-key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" + permissions = "/go/src/github.com/molecula/featurebase/internal/authclustertests/testdata/permissions.yaml" + query-log-path = "query-log-test.log" + redirect-base-url = "https://localhost:10101" diff --git a/internal/authclustertests/testdata/permissions.yaml b/internal/authclustertests/testdata/permissions.yaml new file mode 100644 index 000000000..d5af09bed --- /dev/null +++ b/internal/authclustertests/testdata/permissions.yaml @@ -0,0 +1,4 @@ +user-groups: + "group-id-test": + "test": "write" +admin: "group-id-test" diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go index 4bcdb2c58..4ad532e8a 100644 --- a/internal/clustertests/cluster_test.go +++ b/internal/clustertests/cluster_test.go @@ -11,9 +11,12 @@ import ( "testing" "time" + "github.com/golang-jwt/jwt" pilosa "github.com/molecula/featurebase/v3" + "github.com/molecula/featurebase/v3/authn" "github.com/molecula/featurebase/v3/disco" picli "github.com/molecula/featurebase/v3/http" + "github.com/molecula/featurebase/v3/logger" ) // container turns a docker-compose service name into a container name @@ -24,16 +27,66 @@ import ( // as well, but I think it is true in recent versions. func container(svc string) string { project := "clustertests" + if os.Getenv("ENABLE_AUTH") == "1" { + project = "authclustertests" + } + if p := os.Getenv("PROJECT"); p != "" { project = p } return project + "_" + svc + "_1" } +func GetAuthToken(t *testing.T) string { + t.Helper() + var ( + ClientID = "e9088663-eb08-41d7-8f65-efb5f54bbb71" + ClientSecret = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" + AuthorizeURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/authorize" + TokenURL = "https://login.microsoftonline.com/4a137d66-d161-4ae4-b1e6-07e9920874b8/oauth2/v2.0/token" + GroupEndpointURL = "https://graph.microsoft.com/v1.0/me/transitiveMemberOf/microsoft.graph.group?$count=true" + LogoutURL = "https://login.microsoftonline.com/common/oauth2/v2.0/logout" + Scopes = []string{"https://graph.microsoft.com/.default", "offline_access"} + Key = "DEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEFDEADBEEF" + ) + + a, err := authn.NewAuth( + logger.NewStandardLogger(os.Stdout), + "http://localhost:10101/", + Scopes, + AuthorizeURL, + TokenURL, + GroupEndpointURL, + LogoutURL, + ClientID, + ClientSecret, + Key, + ) + + // make a valid token + tkn := jwt.New(jwt.SigningMethodHS256) + claims := tkn.Claims.(jwt.MapClaims) + groupString, _ := authn.ToGob64([]authn.Group{{GroupID: "group-id-test", GroupName: "group-name-test"}}) + claims["molecula-idp-groups"] = groupString + claims["oid"] = "42" + claims["name"] = "valid" + token, err := tkn.SignedString([]byte(a.SecretKey())) + if err != nil { + t.Fatal(err) + } + + return token +} func TestClusterStuff(t *testing.T) { if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" { t.Skip("pilosa cluster tests are not enabled") } + + auth := false + if os.Getenv("ENABLE_AUTH") == "1" { + auth = true + } + cli1, err := picli.NewInternalClient("pilosa1:10101", picli.GetHTTPClient(nil)) if err != nil { t.Fatalf("getting client: %v", err) @@ -46,11 +99,18 @@ func TestClusterStuff(t *testing.T) { if err != nil { t.Fatalf("getting client: %v", err) } + ctx := context.Background() + token := "" + // generate auth token and add to context + if auth { + token = GetAuthToken(t) + ctx = context.WithValue(ctx, "token", "Bearer "+token) + } - if err := cli1.CreateIndex(context.Background(), "testidx", pilosa.IndexOptions{}); err != nil { + if err := cli1.CreateIndex(ctx, "testidx", pilosa.IndexOptions{}); err != nil { t.Fatalf("creating index: %v", err) } - if err := cli1.CreateFieldWithOptions(context.Background(), "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100}); err != nil { + if err := cli1.CreateFieldWithOptions(ctx, "testidx", "testf", pilosa.FieldOptions{CacheType: pilosa.CacheTypeRanked, CacheSize: 100}); err != nil { t.Fatalf("creating field: %v", err) } @@ -66,7 +126,7 @@ func TestClusterStuff(t *testing.T) { req.ColumnIDs[i%10] = uint64((i/10)*pilosa.ShardWidth + i%10) req.Shard = uint64(i / 10) if i%10 == 9 { - err = cli1.Import(context.Background(), nil, req, &pilosa.ImportOptions{}) + err = cli1.Import(ctx, nil, req, &pilosa.ImportOptions{}) if err != nil { t.Fatalf("importing: %v", err) } @@ -75,7 +135,7 @@ func TestClusterStuff(t *testing.T) { // Check query results from each node. for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} { - r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) + r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) if err != nil { t.Fatalf("count querying pilosa%d: %v", i, err) } @@ -97,12 +157,12 @@ func TestClusterStuff(t *testing.T) { } t.Log("done with pause, waiting for stability") - waitForStatus(t, cli1.Status, string(disco.ClusterStateNormal), 30, time.Second) + waitForStatus(t, cli1.Status, string(disco.ClusterStateNormal), 30, time.Second, ctx) t.Log("done waiting for stability") // Check query results from each node. for i, cli := range []*picli.InternalClient{cli1, cli2, cli3} { - r, err := cli.Query(context.Background(), "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) + r, err := cli.Query(ctx, "testidx", &pilosa.QueryRequest{Index: "testidx", Query: "Count(Row(testf=0))"}) if err != nil { t.Fatalf("count querying pilosa%d: %v", i, err) } @@ -119,9 +179,17 @@ func TestClusterStuff(t *testing.T) { } var backupCmd *exec.Cmd tmpdir := t.TempDir() - if backupCmd, err = startCmd( - "featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest")); err != nil { - t.Fatalf("sending backup command: %v", err) + + if auth { + if backupCmd, err = startCmd( + "featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest"), "--auth-token", token); err != nil { + t.Fatalf("sending backup command: %v", err) + } + } else { + if backupCmd, err = startCmd( + "featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest")); err != nil { + t.Fatalf("sending backup command: %v", err) + } } time.Sleep(time.Second * 5) if err = sendCmd("docker", "start", container("pilosa1")); err != nil { @@ -133,7 +201,11 @@ func TestClusterStuff(t *testing.T) { } client := http.Client{} - if req, err := http.NewRequest(http.MethodDelete, "http://pilosa1:10101/index/testidx", nil); err != nil { + req, err := http.NewRequest(http.MethodDelete, "http://pilosa1:10101/index/testidx", nil) + if auth { + req.Header.Set("Authorization", "Bearer "+token) + } + if err != nil { t.Fatalf("getting req: %v", err) } else if resp, err := client.Do(req); err != nil { t.Fatalf("doing request: %v", err) @@ -146,8 +218,14 @@ func TestClusterStuff(t *testing.T) { } var restoreCmd *exec.Cmd - if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101"); err != nil { - t.Fatalf("starting restore: %v", err) + if auth { + if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101", "--auth-token", token); err != nil { + t.Fatalf("starting restore: %v", err) + } + } else { + if restoreCmd, err = startCmd("featurebase", "restore", "-s", tmpdir+"/backuptest", "--host", "pilosa1:10101"); err != nil { + t.Fatalf("starting restore: %v", err) + } } time.Sleep(time.Millisecond * 50) if err = sendCmd("docker", "stop", container("pilosa2")); err != nil { @@ -165,9 +243,16 @@ func TestClusterStuff(t *testing.T) { // now do backup with all nodes down and too short a timeout // so it fails. Has be to be all 3 because the cluster has // replicas=3 and the backup command will retry on replicas. - if backupCmd, err = startCmd( - "featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms"); err != nil { - t.Fatalf("sending second backup command: %v", err) + if auth { + if backupCmd, err = startCmd( + "featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms", "--auth-token", token); err != nil { + t.Fatalf("sending second backup command: %v", err) + } + } else { + if backupCmd, err = startCmd( + "featurebase", "backup", "--host=pilosa1:10101", fmt.Sprintf("--output=%s", tmpdir+"/backuptest2"), "--retry-period=200ms"); err != nil { + t.Fatalf("sending second backup command: %v", err) + } } time.Sleep(time.Millisecond * 10) // want the backup to get started, then fail if err = sendCmd("docker", "stop", container("pilosa1")); err != nil { @@ -198,11 +283,11 @@ func TestClusterStuff(t *testing.T) { }) } -func waitForStatus(t *testing.T, stator func(context.Context) (string, error), status string, n int, sleep time.Duration) { +func waitForStatus(t *testing.T, stator func(context.Context) (string, error), status string, n int, sleep time.Duration, ctx context.Context) { t.Helper() for i := 0; i < n; i++ { - s, err := stator(context.TODO()) + s, err := stator(ctx) if err != nil { t.Logf("Status (try %d/%d): %v (retrying in %s)", i, n, err, sleep.String()) } else { @@ -214,7 +299,7 @@ func waitForStatus(t *testing.T, stator func(context.Context) (string, error), s time.Sleep(sleep) } - s, err := stator(context.TODO()) + s, err := stator(ctx) if err != nil { t.Fatalf("querying status: %v", err) } diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml index 4154850dc..0454035c9 100644 --- a/internal/clustertests/docker-compose.yml +++ b/internal/clustertests/docker-compose.yml @@ -65,6 +65,7 @@ services: - ENABLE_PILOSA_CLUSTER_TESTS=1 - GO111MODULE=on - PROJECT=${PROJECT} + - ENABLE_AUTH=0 networks: - pilosanet volumes: diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go index 20613ff61..af053a6bf 100644 --- a/internal/clustertests/pause_node_test.go +++ b/internal/clustertests/pause_node_test.go @@ -270,6 +270,12 @@ func TestPauseReplica(t *testing.T) { if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" { t.Skip("pilosa cluster tests for replication when a replica is paused are not enabled") } + + auth := false + if os.Getenv("ENABLE_AUTH") == "1" { + auth = true + } + // configurations for test nodeNames := []string{"pilosa1", "pilosa2", "pilosa3"} nodeToPause := "pilosa3" @@ -289,12 +295,17 @@ func TestPauseReplica(t *testing.T) { uri := uris[0] ctx := context.Background() + if auth { + token := GetAuthToken(t) + ctx = context.WithValue(ctx, "token", "Bearer "+token) + } + ctx, cancel := context.WithCancel(ctx) t.Log("start Client") // first achieve normal cluster status - waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second) + waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second, ctx) // create keyed index rng := rand.New(rand.NewSource(time.Now().UnixNano())) @@ -354,7 +365,7 @@ func TestPauseReplica(t *testing.T) { t.Logf("successfully end insert: %v", len(ts)) // wait for cluster status to be non-normal - waitForStatus(t, cli.Status, string(disco.ClusterStateDegraded), 30, 1*time.Second) + waitForStatus(t, cli.Status, string(disco.ClusterStateDegraded), 30, 1*time.Second, ctx) // wait for cluster status to get back to normal t.Logf("unpause %s", nodeToPause) @@ -362,7 +373,7 @@ func TestPauseReplica(t *testing.T) { if err != nil { t.Fatalf("error on unpause node %s: %v", nodeToPause, err) } - waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second) + waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second, ctx) // set up directory to store keys basePath := "."