diff --git a/.circleci/config.yml b/.circleci/config.yml index 498c56ea3..c3e1a43e4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -144,16 +144,6 @@ jobs: - run: command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable no_output_timeout: 30m - test-backup-restore: - executor: - name: golang - steps: - - checkout-plus - - skip-if-root-unchanged - - setup_remote_docker - - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - - run: make backuptests-build - - run: make backuptests cluster-tests: executor: name: golang @@ -273,10 +263,6 @@ workflows: context: molecula requires: - setup - - test-backup-restore: - context: molecula - requires: - - setup - cluster-tests: context: molecula requires: @@ -292,8 +278,6 @@ workflows: filters: tags: only: /^v.*/ - branches: - only: master - publish_release: context: molecula requires: diff --git a/Dockerfile.pilosa b/Dockerfile.pilosa deleted file mode 100644 index 6ee2e0bbf..000000000 --- a/Dockerfile.pilosa +++ /dev/null @@ -1,37 +0,0 @@ -ARG GO_VERSION=latest - -###################### -### Pilosa builder ### -###################### - -FROM golang:${GO_VERSION} as pilosa-builder -ARG MAKE_FLAGS -WORKDIR /pilosa - -COPY . ./ - -RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS} - -##################### -### Pilosa runner ### -##################### - -FROM alpine:3.13.2 as runner - -LABEL maintainer "dev@molecula.com" - -RUN apk add --no-cache curl jq - -COPY --from=pilosa-builder /pilosa/build/featurebase / - -COPY NOTICE /NOTICE - -EXPOSE 10101 -VOLUME /data - -ENV PILOSA_DATA_DIR /data -ENV PILOSA_BIND 0.0.0.0:10101 -ENV PILOSA_BIND_GRPC 0.0.0.0:20101 - -ENTRYPOINT ["/featurebase"] -CMD ["server"] diff --git a/Dockerfile.runner b/Dockerfile.runner deleted file mode 100644 index 12f49e13c..000000000 --- a/Dockerfile.runner +++ /dev/null @@ -1,23 +0,0 @@ -ARG GO_VERSION=latest - -###################### -### Pilosa builder ### -###################### - -FROM golang:${GO_VERSION} as pilosa-builder -ARG MAKE_FLAGS -WORKDIR /pilosa - -COPY . ./ - -RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS} - -FROM moleculacorp/idk as idk -LABEL maintainer "dev@molecula.com" -RUN apt-get update -y -RUN apt-get install -y bash curl jq - - -COPY --from=pilosa-builder /pilosa/build/featurebase / -COPY testBackupRestore.sh / -CMD ["bash","/testBackupRestore.sh"] diff --git a/Makefile b/Makefile index f72517684..a24d8c40a 100644 --- a/Makefile +++ b/Makefile @@ -155,15 +155,6 @@ clustertests: vendor clustertests-build: vendor docker-compose -f $(DOCKER_COMPOSE) down -v docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build -# Test Cluster backup and restore -backuptests-build: vendor - docker-compose -f docker-compose-3.yml down - docker-compose -f docker-compose-3.yml build - -backuptests: vendor - docker-compose -f docker-compose-3.yml down -v - docker-compose -f docker-compose-3.yml up --exit-code-from=client1 --abort-on-container-exit - # Install Pilosa install: diff --git a/api.go b/api.go index 07e971599..d39a61f3d 100644 --- a/api.go +++ b/api.go @@ -834,6 +834,15 @@ func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName return f, nil } +type RedirectError struct { + HostPort string + error string +} + +func (r RedirectError) Error() string { + return r.error +} + // TranslateData returns all translation data in the specified partition. func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error) { span, _ := tracing.StartSpanFromContext(ctx, "API.TranslateData") @@ -849,6 +858,15 @@ func (api *API) TranslateData(ctx context.Context, indexName string, partition i return nil, newNotFoundError(ErrIndexNotFound, indexName) } + snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN) + nodes := snap.PartitionNodes(partition) + if nodes[0].ID != api.server.NodeID() { + return nil, RedirectError{ + HostPort: nodes[0].URI.HostPort(), + error: fmt.Sprintf("can't translate data, this node(%s) does not partition %d", api.server.uri, partition), + } + } + // Retrieve translatestore from holder. store := idx.TranslateStore(partition) if store == nil { @@ -978,7 +996,10 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e return resp, nil } - if api.usageCache.lastCalcDuration < usageCacheMinDuration { + api.usageCache.muAssign.Lock() + lastCalc := api.usageCache.lastCalcDuration + api.usageCache.muAssign.Unlock() + if lastCalc < usageCacheMinDuration { err := api.ResetUsageCache() if err != nil { api.server.logger.Infof("could not reset usageCache: %s", err) diff --git a/client/client.go b/client/client.go index 5fd3efa9c..211e7c3f7 100644 --- a/client/client.go +++ b/client/client.go @@ -808,7 +808,6 @@ func (c *Client) shardsMax() (map[string]uint64, error) { } // HTTPRequest sends an HTTP request to the Pilosa server (used by idk) -// nolint: deadcode func (c *Client) HTTPRequest(method string, path string, data []byte, headers map[string]string) (status int, body []byte, err error) { span := c.tracer.StartSpan("Client.HTTPRequest") diff --git a/ctl/backup.go b/ctl/backup.go index 6e7e52e83..0e8a257f1 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -18,7 +18,7 @@ import ( "golang.org/x/sync/errgroup" ) -// BackupCommand represents a command for backing up a Pilosa node. +// BackupCommand represents a command for backing up a FeatureBase node. type BackupCommand struct { // nolint: maligned tlsConfig *tls.Config @@ -183,12 +183,17 @@ func (cmd *BackupCommand) backupIDAllocData(ctx context.Context) error { func (cmd *BackupCommand) backupIndexTranslation(ctx context.Context, ii *pilosa.IndexInfo) error { logger := cmd.Logger() logger.Printf("backing up index translation: %q", ii.Name) - if err := cmd.backupIndexTranslateData(ctx, ii.Name); err != nil { - return err + if ii.Options.Keys { + if err := cmd.backupIndexTranslateData(ctx, ii.Name); err != nil { + return err + } } // Back up field translation data. for _, fi := range ii.Fields { + if !fi.Options.Keys { + continue + } if err := cmd.backupFieldTranslateData(ctx, ii.Name, fi.Name); err != nil { return fmt.Errorf("cannot backup field translation data for field %q on index %q: %w", fi.Name, ii.Name, err) } @@ -286,7 +291,6 @@ func (cmd *BackupCommand) backupShardNode(ctx context.Context, indexName string, func (cmd *BackupCommand) backupIndexTranslateData(ctx context.Context, name string) error { partitionN := topology.DefaultPartitionN - // Back up all bitmap data for the index. ch := make(chan int, partitionN) for partitionID := 0; partitionID < partitionN; partitionID++ { ch <- partitionID @@ -318,9 +322,7 @@ func (cmd *BackupCommand) backupIndexPartitionTranslateData(ctx context.Context, logger.Printf("backing up index translation data: %s/%d", name, partitionID) rc, err := cmd.client.IndexTranslateDataReader(ctx, name, partitionID) - if err == pilosa.ErrTranslateStoreNotFound { - return nil - } else if err != nil { + if err != nil { return fmt.Errorf("fetching translate data reader: %w", err) } defer rc.Close() @@ -349,9 +351,7 @@ func (cmd *BackupCommand) backupFieldTranslateData(ctx context.Context, indexNam logger.Printf("backing up field translation data: %s/%s", indexName, fieldName) rc, err := cmd.client.FieldTranslateDataReader(ctx, indexName, fieldName) - if err == pilosa.ErrTranslateStoreNotFound { - return nil - } else if err != nil { + if err != nil { return fmt.Errorf("fetching translate data reader: %w", err) } defer rc.Close() diff --git a/ctl/chksum.go b/ctl/chksum.go index 5514fc362..af2430efb 100644 --- a/ctl/chksum.go +++ b/ctl/chksum.go @@ -8,7 +8,7 @@ import ( "io" "github.com/cespare/xxhash" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/server" ) @@ -61,14 +61,19 @@ func (cmd *ChkSumCommand) Run(ctx context.Context) (err error) { h := xxhash.New() for _, ii := range schema.Indexes { - qa := &pilosa.QueryRequest{Index: ii.Name, Query: "Count(All())"} + qa := &pilosa.QueryRequest{Index: ii.Name, Query: "All()"} rs, err := client.Query(ctx, ii.Name, qa) if err != nil { return err } - all := rs.Results[0].(uint64) - as := fmt.Sprintf("all=%v", all) - _, _ = h.Write([]byte(as)) + + all := rs.Results[0].(*pilosa.Row) + if len(all.Keys) > 0 { + allString := fmt.Sprintf("%v", all.Keys) + _, _ = h.Write([]byte(allString)) + } else { + _, _ = h.Write(all.Roaring()) + } for _, field := range ii.Fields { switch field.Options.Type { @@ -92,7 +97,7 @@ func (cmd *ChkSumCommand) Run(ctx context.Context) (err error) { } for _, item := range res.Results { rowids := item.(*pilosa.RowIdentifiers) - //either rowids or keys + // either rowids or keys for _, row := range rowids.Keys { countPql := fmt.Sprintf(`Count(Row(%v="%v"))`, field.Name, row) qr := &pilosa.QueryRequest{Index: ii.Name, Query: countPql} @@ -121,7 +126,7 @@ func (cmd *ChkSumCommand) Run(ctx context.Context) (err error) { } } - fmt.Printf("hash:%x\n", h.Sum(nil)) + fmt.Fprintf(cmd.Stdout, "hash:%x\n", h.Sum(nil)) } return nil diff --git a/docker-compose-3.yml b/docker-compose-3.yml deleted file mode 100644 index ef31424d0..000000000 --- a/docker-compose-3.yml +++ /dev/null @@ -1,97 +0,0 @@ -version: "3" -services: - pilosa0: - image: build/pilosa - build: - context: . - dockerfile: Dockerfile.pilosa - environment: - PILOSA_ADVERTISE: pilosa0:10101 - PILOSA_ADVERTISE_GRPC: pilosa0:20101 - PILOSA_CLUSTER_REPLICAS: 1 - PILOSA_DATA_DIR: /data/pilosa0 - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosa0:10201 - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosa0:10301 - PILOSA_ETCD_INITIAL_CLUSTER: pilosa0=http://pilosa0:10301,pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301 - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201 - PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301 - PILOSA_NAME: pilosa0 - PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf} - volumes: - - data:/data - healthcheck: - test: x=$$(curl -s localhost:10101/status | jq -r ".state") && [[ "$$x" == "NORMAL" ]] || $$(exit 1) - interval: 10s - timeout: 5s - retries: 5 - pilosa1: - image: build/pilosa - build: - context: . - dockerfile: Dockerfile.pilosa - environment: - PILOSA_ADVERTISE: pilosa1:10101 - PILOSA_ADVERTISE_GRPC: pilosa1:20101 - PILOSA_CLUSTER_REPLICAS: 1 - PILOSA_DATA_DIR: /data/pilosa1 - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosa1:10201 - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosa1:10301 - PILOSA_ETCD_INITIAL_CLUSTER: pilosa0=http://pilosa0:10301,pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301 - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201 - PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301 - PILOSA_NAME: pilosa1 - PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf} - volumes: - - data:/data - pilosa2: - image: build/pilosa - build: - context: . - dockerfile: Dockerfile.pilosa - environment: - PILOSA_ADVERTISE: pilosa2:10101 - PILOSA_ADVERTISE_GRPC: pilosa2:20101 - PILOSA_CLUSTER_REPLICAS: 1 - PILOSA_DATA_DIR: /data/pilosa2 - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosa2:10201 - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosa2:10301 - PILOSA_ETCD_INITIAL_CLUSTER: pilosa0=http://pilosa0:10301,pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301 - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201 - PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301 - PILOSA_NAME: pilosa2 - PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf} - volumes: - - data:/data - pilosax: - image: build/pilosa - build: - context: . - dockerfile: Dockerfile.pilosa - environment: - PILOSA_ADVERTISE: pilosax:10101 - PILOSA_ADVERTISE_GRPC: pilosax:20101 - PILOSA_CLUSTER_REPLICAS: 1 - PILOSA_DATA_DIR: /data/pilosax - PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosax:10201 - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosax:10301 - PILOSA_ETCD_INITIAL_CLUSTER: pilosax=http://pilosax:10301 - PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201 - PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301 - PILOSA_NAME: pilosax - PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf} - volumes: - - data:/data - client1: - image: tgruben/bash - build: - context: . - dockerfile: Dockerfile.runner - environment: - - GO111MODULE=on - volumes: - - /var/run/docker.sock:/var/run/docker.sock - depends_on: - - pilosa0 - -volumes: - data: diff --git a/executor_test.go b/executor_test.go index 56023107e..432499511 100644 --- a/executor_test.go +++ b/executor_test.go @@ -27,6 +27,7 @@ import ( "github.com/google/go-cmp/cmp/cmpopts" pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" + "github.com/molecula/featurebase/v2/ctl" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/pql" @@ -6748,7 +6749,7 @@ func variousQueriesCountDistinctTimestamp(t *testing.T, c *test.Cluster) { field := "ts" // create an index and timestamp field - c.CreateField(t, index, pilosa.IndexOptions{}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s")) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s")) // add some data data := []string{"2010-01-02T12:32:00Z", "2010-04-20T12:32:00Z", "2011-04-20T12:32:00Z"} @@ -7015,19 +7016,90 @@ func TestMissingKeyRegression(t *testing.T) { // (single and multi-node clusters, different endpoints for the // queries (HTTP, GRPC, Postgres), etc.). func TestVariousQueries(t *testing.T) { - for _, clusterSize := range []int{1, 3, 4, 7} { + for _, clusterSize := range []int{1, 3, 5} { clusterSize := clusterSize t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) { c := test.MustRunCluster(t, clusterSize) defer c.Close() + + // put a variety of data into the cluster + populateTestData(t, c) + backupTest(t, c, usersIndex) + variousQueries(t, c) variousQueriesOnTimeFields(t, c) variousQueriesOnPercentiles(t, c) variousQueriesCountDistinctTimestamp(t, c) + backupTest(t, c, "") // test backup/restore of all indexes }) } } +func backupTest(t *testing.T, c *test.Cluster, index string) { + // should this really be in executor? No. But all these + // integration-y query tests probably shouldn't be either. My goal + // putting this here is to take advantage of already-existing + // clusters and data. + + sum := chkSumCluster(t, c) + + backupDir := backupCluster(t, c, index) + + cnew := test.MustRunCluster(t, 3) // this way we test 1->3 3->3 5->3 + defer cnew.Close() + + restoreCluster(t, backupDir, cnew) + + sumNew := chkSumCluster(t, cnew) + + if sum != sumNew { + t.Fatalf("old/new checksum mismatch, old:\n%s\nnew:\n%s", sum, sumNew) + } +} + +func chkSumCluster(t *testing.T, c *test.Cluster) string { + buf := &bytes.Buffer{} + + chkSum := ctl.NewChkSumCommand(nil, buf, buf) + chkSum.Host = c.Nodes[len(c.Nodes)-1].URL() + if err := chkSum.Run(context.Background()); err != nil { + t.Fatalf("running checksum: %v", err) + } + + return buf.String() +} + +func backupCluster(t *testing.T, c *test.Cluster, index string) (backupDir string) { + td, err := testhook.TempDir(t, "backupTest") + if err != nil { + t.Fatalf("can't even get a temp dir, what a ripoff: %v", err) + } + td = td + "/backupTest" + + buf := &bytes.Buffer{} + backupCommand := ctl.NewBackupCommand(nil, buf, buf) + backupCommand.Host = c.Nodes[len(c.Nodes)-1].URL() // don't pick node 0 so we don't always get primary (better code coverage) + backupCommand.Index = index + backupCommand.OutputDir = td + + if err := backupCommand.Run(context.Background()); err != nil { + t.Log(buf.String()) + t.Fatalf("running backup: %v", err) + } + return td +} + +func restoreCluster(t *testing.T, backupDir string, c *test.Cluster) { + buf := &bytes.Buffer{} + + restore := ctl.NewRestoreCommand(nil, buf, buf) + restore.Host = c.Nodes[len(c.Nodes)-1].URL() + restore.Path = backupDir + if err := restore.Run(context.Background()); err != nil { + t.Fatalf("restoring: %v", err) + } +} + // tests for abbreviating time values in queries func variousQueriesOnPercentiles(t *testing.T, c *test.Cluster) { // todo, make rand more random, 42 isnt the answer to everything @@ -7332,10 +7404,12 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) { } } -func variousQueries(t *testing.T, c *test.Cluster) { +var usersIndex = "users" + +func populateTestData(t *testing.T, c *test.Cluster) { // Create and populate "likenums" similar to "likes", but without keys on the field. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums") - c.ImportIDKey(t, "users", "likenums", []test.KeyID{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums") + c.ImportIDKey(t, usersIndex, "likenums", []test.KeyID{ {ID: 1, Key: "userA"}, {ID: 2, Key: "userB"}, {ID: 3, Key: "userC"}, @@ -7353,8 +7427,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "likes" field. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys()) - c.ImportKeyKey(t, "users", "likes", [][2]string{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys()) + c.ImportKeyKey(t, usersIndex, "likes", [][2]string{ {"molecula", "userA"}, {"pilosa", "userB"}, {"pangolin", "userC"}, @@ -7370,8 +7444,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "dinner" field. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys()) - c.ImportKeyKey(t, "users", "dinner", [][2]string{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys()) + c.ImportKeyKey(t, usersIndex, "dinner", [][2]string{ {"leftovers", "userB"}, {"pizza", "userA"}, {"pizza", "userB"}, @@ -7381,11 +7455,11 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "places_visited" time field. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"))) + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM"))) ts2019Jan01 := int64(1546300800) * 1e+9 // 2019 January 1st 0:00:00 ts2019Aug01 := int64(1564617600) * 1e+9 // 2019 August 1st 0:00:00 ts2020Jan01 := int64(1577836800) * 1e+9 // 2020 January 1st 0:00:00 - c.ImportTimeQuantumKey(t, "users", "places_visited", []test.TimeQuantumKey{ + c.ImportTimeQuantumKey(t, usersIndex, "places_visited", []test.TimeQuantumKey{ // 2019 January: nairobi, paris, austin, toronto {RowKey: "nairobi", ColKey: "userB", Ts: ts2019Jan01}, {RowKey: "paris", ColKey: "userC", Ts: ts2019Jan01}, @@ -7405,8 +7479,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "affinity" int field with negative, positive, zero and null values. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000)) - c.ImportIntKey(t, "users", "affinity", []test.IntKey{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000)) + c.ImportIntKey(t, usersIndex, "affinity", []test.IntKey{ {Val: 10, Key: "userA"}, {Val: -10, Key: "userB"}, {Val: 5, Key: "userC"}, @@ -7415,8 +7489,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { }) // Create and populate "net_worth" int field with positive values. - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000)) - c.ImportIntKey(t, "users", "net_worth", []test.IntKey{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000)) + c.ImportIntKey(t, usersIndex, "net_worth", []test.IntKey{ {Val: 1, Key: "userA"}, {Val: 10, Key: "userB"}, {Val: 100, Key: "userC"}, @@ -7425,8 +7499,8 @@ func variousQueries(t *testing.T, c *test.Cluster) { {Val: 100000, Key: "userF"}, }) - c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000)) - c.ImportIntKey(t, "users", "zip_code", []test.IntKey{ + c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000)) + c.ImportIntKey(t, usersIndex, "zip_code", []test.IntKey{ {Val: 78739, Key: "userA"}, {Val: 78739, Key: "userB"}, {Val: 19707, Key: "userC"}, @@ -7434,7 +7508,12 @@ func variousQueries(t *testing.T, c *test.Cluster) { {Val: 86753, Key: "userE"}, {Val: 78739, Key: "userG"}, }) +} +func variousQueries(t *testing.T, c *test.Cluster) { + // NOTE: this relies on populateTestData being called first + + // define and run a bunch of tests tests := []struct { query string qrVerifier func(t *testing.T, resp pilosa.QueryResponse) @@ -7781,8 +7860,8 @@ leftovers,1 for i, tst := range tests { t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) { - resp := c.Query(t, "users", tst.query) - tr := c.QueryGRPC(t, "users", tst.query) + resp := c.Query(t, usersIndex, tst.query) + tr := c.QueryGRPC(t, usersIndex, tst.query) if tst.qrVerifier != nil { tst.qrVerifier(t, resp) } @@ -8352,7 +8431,7 @@ func MinMaxTimestampNodeTester(t *testing.T, numNodes int) { defer c.Close() // create an index and timestamp field - c.CreateField(t, index, pilosa.IndexOptions{}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s")) + c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s")) // add some data expected := "2010-01-02T12:32:00Z" diff --git a/field.go b/field.go index 7076c6e98..2aa865a1f 100644 --- a/field.go +++ b/field.go @@ -1884,7 +1884,7 @@ func applyDefaultOptions(o *FieldOptions) FieldOptions { // are included. func (o *FieldOptions) MarshalJSON() ([]byte, error) { switch o.Type { - case FieldTypeSet: + case FieldTypeSet, "": return json.Marshal(struct { Type string `json:"type"` CacheType string `json:"cacheType"` @@ -1975,7 +1975,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) { o.Type, }) } - return nil, errors.New("invalid field type") + return nil, errors.Errorf("invalid field type: '%s'", o.Type) } // MinTimestamp returns the minimum value for a timestamp field. diff --git a/http/handler.go b/http/handler.go index 94665bbfa..9e626babf 100644 --- a/http/handler.go +++ b/http/handler.go @@ -2326,6 +2326,12 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request) // Retrieve partition data from holder. p, err := h.api.TranslateData(r.Context(), q.Get("index"), int(partition)) + if redir, ok := err.(pilosa.RedirectError); ok { + newURL := *r.URL + newURL.Host = redir.HostPort + http.Redirect(w, r, newURL.String(), http.StatusSeeOther) + return + } if err != nil { http.Error(w, err.Error(), http.StatusNotFound) return diff --git a/testBackupRestore.sh b/testBackupRestore.sh deleted file mode 100755 index 7bcbe3894..000000000 --- a/testBackupRestore.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash - -set -eux - -declare STATUS="NORMAL" -declare TIMEOUT=30 -sleep 4 -STATUS=$STATUS timeout -s TERM $TIMEOUT bash -c \ - 'while [[ ${STATUS_RECEIVED} != ${STATUS} ]];\ - do STATUS_RECEIVED=$(curl --connect-timeout 1 -s pilosa0:10101/status | jq -r ".state") && \ - echo "received status: $STATUS_RECEIVED" && \ - sleep 1;\ - done;' -echo "NOW DO STUFF" -datagen --source kitchensink_keyed -e 9999 --pilosa.index sink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101 -before=$(/featurebase chksum --host pilosa0:10101) -/featurebase backup -o backupdir --host pilosa0:10101 -curl -X DELETE -s pilosa0:10101/index/sink -/featurebase restore -s backupdir --host pilosa0:10101 -after=$(/featurebase chksum --host pilosa0:10101) -if [ "$before" = "$after" ]; then - echo "PASS Cluster" -else - echo "FAIL Single" - exit 1 -fi -/featurebase restore -s backupdir --host pilosax:10101 -single=$(/featurebase chksum --host pilosax:10101) -if [ "$before" = "$single" ]; then - echo "PASS Single" - exit 0 -else - echo "FAIL Single" - exit 1 -fi - -datagen --source texas_health -e 9999 --pilosa.index newsink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101 -before=$(/featurebase chksum --host pilosa0:10101) -/featurebase backup -o newbackupdir --host pilosa0:10101 --index newsink -curl -X DELETE -s pilosa0:10101/index/newsink -/featurebase restore -s newbackupdir --host pilosa0:10101 -after=$(/featurebase chksum --host pilosa0:10101) -if [ "$before" = "$after" ]; then - echo "PASS Cluster Table" -else - echo "FAIL Single Table" - exit 1 -fi -/featurebase restore -s newbackupdir --host pilosax:10101 -single=$(/featurebase chksum --host pilosax:10101) -if [ "$before" = "$single" ]; then - echo "PASS Single Table" - exit 0 -else - echo "FAIL Single Table" - exit 1 -fi diff --git a/translate.go b/translate.go index a9d547cb7..be1c47306 100644 --- a/translate.go +++ b/translate.go @@ -568,6 +568,8 @@ func (s *InMemTranslateStore) WriteTo(w io.Writer) (int64, error) { // don't expect to use InMemTranslateStore much, it's mostly there to // avoid disk load during testing. func (s *InMemTranslateStore) ReadFrom(r io.Reader) (count int64, err error) { + s.mu.Lock() + defer s.mu.Unlock() var bytes []byte bytes, err = ioutil.ReadAll(r) count = int64(len(bytes))