mirror of
https://github.com/featurebasedb/featurebase.git
synced 2026-10-09 12:27:52 +00:00
Merge pull request #1798 from molecula/fix-backup-key-translation
FB-1080 Fix backup key translation
This commit is contained in:
Matthew Jaffee
GitHub
commit
9283d52141
14 changed files with 153 additions and 280 deletions
|
|
@ -144,16 +144,6 @@ jobs:
|
|||
- run:
|
||||
command: make test-external-lookup EXTERNAL_LOOKUP_DSN=postgresql://postgres:password@localhost/circle_test?sslmode=disable
|
||||
no_output_timeout: 30m
|
||||
test-backup-restore:
|
||||
executor:
|
||||
name: golang
|
||||
steps:
|
||||
- checkout-plus
|
||||
- skip-if-root-unchanged
|
||||
- setup_remote_docker
|
||||
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
|
||||
- run: make backuptests-build
|
||||
- run: make backuptests
|
||||
cluster-tests:
|
||||
executor:
|
||||
name: golang
|
||||
|
|
@ -273,10 +263,6 @@ workflows:
|
|||
context: molecula
|
||||
requires:
|
||||
- setup
|
||||
- test-backup-restore:
|
||||
context: molecula
|
||||
requires:
|
||||
- setup
|
||||
- cluster-tests:
|
||||
context: molecula
|
||||
requires:
|
||||
|
|
@ -292,8 +278,6 @@ workflows:
|
|||
filters:
|
||||
tags:
|
||||
only: /^v.*/
|
||||
branches:
|
||||
only: master
|
||||
- publish_release:
|
||||
context: molecula
|
||||
requires:
|
||||
|
|
|
|||
|
|
@ -1,37 +0,0 @@
|
|||
ARG GO_VERSION=latest
|
||||
|
||||
######################
|
||||
### Pilosa builder ###
|
||||
######################
|
||||
|
||||
FROM golang:${GO_VERSION} as pilosa-builder
|
||||
ARG MAKE_FLAGS
|
||||
WORKDIR /pilosa
|
||||
|
||||
COPY . ./
|
||||
|
||||
RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
|
||||
|
||||
#####################
|
||||
### Pilosa runner ###
|
||||
#####################
|
||||
|
||||
FROM alpine:3.13.2 as runner
|
||||
|
||||
LABEL maintainer "dev@molecula.com"
|
||||
|
||||
RUN apk add --no-cache curl jq
|
||||
|
||||
COPY --from=pilosa-builder /pilosa/build/featurebase /
|
||||
|
||||
COPY NOTICE /NOTICE
|
||||
|
||||
EXPOSE 10101
|
||||
VOLUME /data
|
||||
|
||||
ENV PILOSA_DATA_DIR /data
|
||||
ENV PILOSA_BIND 0.0.0.0:10101
|
||||
ENV PILOSA_BIND_GRPC 0.0.0.0:20101
|
||||
|
||||
ENTRYPOINT ["/featurebase"]
|
||||
CMD ["server"]
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
ARG GO_VERSION=latest
|
||||
|
||||
######################
|
||||
### Pilosa builder ###
|
||||
######################
|
||||
|
||||
FROM golang:${GO_VERSION} as pilosa-builder
|
||||
ARG MAKE_FLAGS
|
||||
WORKDIR /pilosa
|
||||
|
||||
COPY . ./
|
||||
|
||||
RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
|
||||
|
||||
FROM moleculacorp/idk as idk
|
||||
LABEL maintainer "dev@molecula.com"
|
||||
RUN apt-get update -y
|
||||
RUN apt-get install -y bash curl jq
|
||||
|
||||
|
||||
COPY --from=pilosa-builder /pilosa/build/featurebase /
|
||||
COPY testBackupRestore.sh /
|
||||
CMD ["bash","/testBackupRestore.sh"]
|
||||
9
Makefile
9
Makefile
|
|
@ -155,15 +155,6 @@ clustertests: vendor
|
|||
clustertests-build: vendor
|
||||
docker-compose -f $(DOCKER_COMPOSE) down -v
|
||||
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 --build
|
||||
# Test Cluster backup and restore
|
||||
backuptests-build: vendor
|
||||
docker-compose -f docker-compose-3.yml down
|
||||
docker-compose -f docker-compose-3.yml build
|
||||
|
||||
backuptests: vendor
|
||||
docker-compose -f docker-compose-3.yml down -v
|
||||
docker-compose -f docker-compose-3.yml up --exit-code-from=client1 --abort-on-container-exit
|
||||
|
||||
|
||||
# Install Pilosa
|
||||
install:
|
||||
|
|
|
|||
23
api.go
23
api.go
|
|
@ -834,6 +834,15 @@ func (api *API) FragmentData(ctx context.Context, indexName, fieldName, viewName
|
|||
return f, nil
|
||||
}
|
||||
|
||||
type RedirectError struct {
|
||||
HostPort string
|
||||
error string
|
||||
}
|
||||
|
||||
func (r RedirectError) Error() string {
|
||||
return r.error
|
||||
}
|
||||
|
||||
// TranslateData returns all translation data in the specified partition.
|
||||
func (api *API) TranslateData(ctx context.Context, indexName string, partition int) (io.WriterTo, error) {
|
||||
span, _ := tracing.StartSpanFromContext(ctx, "API.TranslateData")
|
||||
|
|
@ -849,6 +858,15 @@ func (api *API) TranslateData(ctx context.Context, indexName string, partition i
|
|||
return nil, newNotFoundError(ErrIndexNotFound, indexName)
|
||||
}
|
||||
|
||||
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
|
||||
nodes := snap.PartitionNodes(partition)
|
||||
if nodes[0].ID != api.server.NodeID() {
|
||||
return nil, RedirectError{
|
||||
HostPort: nodes[0].URI.HostPort(),
|
||||
error: fmt.Sprintf("can't translate data, this node(%s) does not partition %d", api.server.uri, partition),
|
||||
}
|
||||
}
|
||||
|
||||
// Retrieve translatestore from holder.
|
||||
store := idx.TranslateStore(partition)
|
||||
if store == nil {
|
||||
|
|
@ -978,7 +996,10 @@ func (api *API) Usage(ctx context.Context, remote bool) (map[string]NodeUsage, e
|
|||
return resp, nil
|
||||
}
|
||||
|
||||
if api.usageCache.lastCalcDuration < usageCacheMinDuration {
|
||||
api.usageCache.muAssign.Lock()
|
||||
lastCalc := api.usageCache.lastCalcDuration
|
||||
api.usageCache.muAssign.Unlock()
|
||||
if lastCalc < usageCacheMinDuration {
|
||||
err := api.ResetUsageCache()
|
||||
if err != nil {
|
||||
api.server.logger.Infof("could not reset usageCache: %s", err)
|
||||
|
|
|
|||
|
|
@ -808,7 +808,6 @@ func (c *Client) shardsMax() (map[string]uint64, error) {
|
|||
}
|
||||
|
||||
// HTTPRequest sends an HTTP request to the Pilosa server (used by idk)
|
||||
// nolint: deadcode
|
||||
func (c *Client) HTTPRequest(method string, path string, data []byte, headers map[string]string) (status int, body []byte, err error) {
|
||||
span := c.tracer.StartSpan("Client.HTTPRequest")
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import (
|
|||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// BackupCommand represents a command for backing up a Pilosa node.
|
||||
// BackupCommand represents a command for backing up a FeatureBase node.
|
||||
type BackupCommand struct { // nolint: maligned
|
||||
tlsConfig *tls.Config
|
||||
|
||||
|
|
@ -183,12 +183,17 @@ func (cmd *BackupCommand) backupIDAllocData(ctx context.Context) error {
|
|||
func (cmd *BackupCommand) backupIndexTranslation(ctx context.Context, ii *pilosa.IndexInfo) error {
|
||||
logger := cmd.Logger()
|
||||
logger.Printf("backing up index translation: %q", ii.Name)
|
||||
if err := cmd.backupIndexTranslateData(ctx, ii.Name); err != nil {
|
||||
return err
|
||||
if ii.Options.Keys {
|
||||
if err := cmd.backupIndexTranslateData(ctx, ii.Name); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// Back up field translation data.
|
||||
for _, fi := range ii.Fields {
|
||||
if !fi.Options.Keys {
|
||||
continue
|
||||
}
|
||||
if err := cmd.backupFieldTranslateData(ctx, ii.Name, fi.Name); err != nil {
|
||||
return fmt.Errorf("cannot backup field translation data for field %q on index %q: %w", fi.Name, ii.Name, err)
|
||||
}
|
||||
|
|
@ -286,7 +291,6 @@ func (cmd *BackupCommand) backupShardNode(ctx context.Context, indexName string,
|
|||
func (cmd *BackupCommand) backupIndexTranslateData(ctx context.Context, name string) error {
|
||||
partitionN := topology.DefaultPartitionN
|
||||
|
||||
// Back up all bitmap data for the index.
|
||||
ch := make(chan int, partitionN)
|
||||
for partitionID := 0; partitionID < partitionN; partitionID++ {
|
||||
ch <- partitionID
|
||||
|
|
@ -318,9 +322,7 @@ func (cmd *BackupCommand) backupIndexPartitionTranslateData(ctx context.Context,
|
|||
logger.Printf("backing up index translation data: %s/%d", name, partitionID)
|
||||
|
||||
rc, err := cmd.client.IndexTranslateDataReader(ctx, name, partitionID)
|
||||
if err == pilosa.ErrTranslateStoreNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching translate data reader: %w", err)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
|
@ -349,9 +351,7 @@ func (cmd *BackupCommand) backupFieldTranslateData(ctx context.Context, indexNam
|
|||
logger.Printf("backing up field translation data: %s/%s", indexName, fieldName)
|
||||
|
||||
rc, err := cmd.client.FieldTranslateDataReader(ctx, indexName, fieldName)
|
||||
if err == pilosa.ErrTranslateStoreNotFound {
|
||||
return nil
|
||||
} else if err != nil {
|
||||
if err != nil {
|
||||
return fmt.Errorf("fetching translate data reader: %w", err)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import (
|
|||
"io"
|
||||
|
||||
"github.com/cespare/xxhash"
|
||||
"github.com/molecula/featurebase/v2"
|
||||
pilosa "github.com/molecula/featurebase/v2"
|
||||
"github.com/molecula/featurebase/v2/server"
|
||||
)
|
||||
|
||||
|
|
@ -61,14 +61,19 @@ func (cmd *ChkSumCommand) Run(ctx context.Context) (err error) {
|
|||
h := xxhash.New()
|
||||
|
||||
for _, ii := range schema.Indexes {
|
||||
qa := &pilosa.QueryRequest{Index: ii.Name, Query: "Count(All())"}
|
||||
qa := &pilosa.QueryRequest{Index: ii.Name, Query: "All()"}
|
||||
rs, err := client.Query(ctx, ii.Name, qa)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
all := rs.Results[0].(uint64)
|
||||
as := fmt.Sprintf("all=%v", all)
|
||||
_, _ = h.Write([]byte(as))
|
||||
|
||||
all := rs.Results[0].(*pilosa.Row)
|
||||
if len(all.Keys) > 0 {
|
||||
allString := fmt.Sprintf("%v", all.Keys)
|
||||
_, _ = h.Write([]byte(allString))
|
||||
} else {
|
||||
_, _ = h.Write(all.Roaring())
|
||||
}
|
||||
|
||||
for _, field := range ii.Fields {
|
||||
switch field.Options.Type {
|
||||
|
|
@ -92,7 +97,7 @@ func (cmd *ChkSumCommand) Run(ctx context.Context) (err error) {
|
|||
}
|
||||
for _, item := range res.Results {
|
||||
rowids := item.(*pilosa.RowIdentifiers)
|
||||
//either rowids or keys
|
||||
// either rowids or keys
|
||||
for _, row := range rowids.Keys {
|
||||
countPql := fmt.Sprintf(`Count(Row(%v="%v"))`, field.Name, row)
|
||||
qr := &pilosa.QueryRequest{Index: ii.Name, Query: countPql}
|
||||
|
|
@ -121,7 +126,7 @@ func (cmd *ChkSumCommand) Run(ctx context.Context) (err error) {
|
|||
}
|
||||
|
||||
}
|
||||
fmt.Printf("hash:%x\n", h.Sum(nil))
|
||||
fmt.Fprintf(cmd.Stdout, "hash:%x\n", h.Sum(nil))
|
||||
}
|
||||
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -1,97 +0,0 @@
|
|||
version: "3"
|
||||
services:
|
||||
pilosa0:
|
||||
image: build/pilosa
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.pilosa
|
||||
environment:
|
||||
PILOSA_ADVERTISE: pilosa0:10101
|
||||
PILOSA_ADVERTISE_GRPC: pilosa0:20101
|
||||
PILOSA_CLUSTER_REPLICAS: 1
|
||||
PILOSA_DATA_DIR: /data/pilosa0
|
||||
PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosa0:10201
|
||||
PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosa0:10301
|
||||
PILOSA_ETCD_INITIAL_CLUSTER: pilosa0=http://pilosa0:10301,pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301
|
||||
PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201
|
||||
PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301
|
||||
PILOSA_NAME: pilosa0
|
||||
PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf}
|
||||
volumes:
|
||||
- data:/data
|
||||
healthcheck:
|
||||
test: x=$$(curl -s localhost:10101/status | jq -r ".state") && [[ "$$x" == "NORMAL" ]] || $$(exit 1)
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
pilosa1:
|
||||
image: build/pilosa
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.pilosa
|
||||
environment:
|
||||
PILOSA_ADVERTISE: pilosa1:10101
|
||||
PILOSA_ADVERTISE_GRPC: pilosa1:20101
|
||||
PILOSA_CLUSTER_REPLICAS: 1
|
||||
PILOSA_DATA_DIR: /data/pilosa1
|
||||
PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosa1:10201
|
||||
PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosa1:10301
|
||||
PILOSA_ETCD_INITIAL_CLUSTER: pilosa0=http://pilosa0:10301,pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301
|
||||
PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201
|
||||
PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301
|
||||
PILOSA_NAME: pilosa1
|
||||
PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf}
|
||||
volumes:
|
||||
- data:/data
|
||||
pilosa2:
|
||||
image: build/pilosa
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.pilosa
|
||||
environment:
|
||||
PILOSA_ADVERTISE: pilosa2:10101
|
||||
PILOSA_ADVERTISE_GRPC: pilosa2:20101
|
||||
PILOSA_CLUSTER_REPLICAS: 1
|
||||
PILOSA_DATA_DIR: /data/pilosa2
|
||||
PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosa2:10201
|
||||
PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosa2:10301
|
||||
PILOSA_ETCD_INITIAL_CLUSTER: pilosa0=http://pilosa0:10301,pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301
|
||||
PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201
|
||||
PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301
|
||||
PILOSA_NAME: pilosa2
|
||||
PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf}
|
||||
volumes:
|
||||
- data:/data
|
||||
pilosax:
|
||||
image: build/pilosa
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.pilosa
|
||||
environment:
|
||||
PILOSA_ADVERTISE: pilosax:10101
|
||||
PILOSA_ADVERTISE_GRPC: pilosax:20101
|
||||
PILOSA_CLUSTER_REPLICAS: 1
|
||||
PILOSA_DATA_DIR: /data/pilosax
|
||||
PILOSA_ETCD_ADVERTISE_CLIENT_ADDRESS: http://pilosax:10201
|
||||
PILOSA_ETCD_ADVERTISE_PEER_ADDRESS: http://pilosax:10301
|
||||
PILOSA_ETCD_INITIAL_CLUSTER: pilosax=http://pilosax:10301
|
||||
PILOSA_ETCD_LISTEN_CLIENT_ADDRESS: http://0.0.0.0:10201
|
||||
PILOSA_ETCD_LISTEN_PEER_ADDRESS: http://0.0.0.0:10301
|
||||
PILOSA_NAME: pilosax
|
||||
PILOSA_STORAGE_BACKEND: ${PILOSA_STORAGE_BACKEND:-rbf}
|
||||
volumes:
|
||||
- data:/data
|
||||
client1:
|
||||
image: tgruben/bash
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile.runner
|
||||
environment:
|
||||
- GO111MODULE=on
|
||||
volumes:
|
||||
- /var/run/docker.sock:/var/run/docker.sock
|
||||
depends_on:
|
||||
- pilosa0
|
||||
|
||||
volumes:
|
||||
data:
|
||||
119
executor_test.go
119
executor_test.go
|
|
@ -27,6 +27,7 @@ import (
|
|||
"github.com/google/go-cmp/cmp/cmpopts"
|
||||
pilosa "github.com/molecula/featurebase/v2"
|
||||
"github.com/molecula/featurebase/v2/boltdb"
|
||||
"github.com/molecula/featurebase/v2/ctl"
|
||||
"github.com/molecula/featurebase/v2/disco"
|
||||
"github.com/molecula/featurebase/v2/http"
|
||||
"github.com/molecula/featurebase/v2/pql"
|
||||
|
|
@ -6748,7 +6749,7 @@ func variousQueriesCountDistinctTimestamp(t *testing.T, c *test.Cluster) {
|
|||
field := "ts"
|
||||
|
||||
// create an index and timestamp field
|
||||
c.CreateField(t, index, pilosa.IndexOptions{}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s"))
|
||||
c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s"))
|
||||
|
||||
// add some data
|
||||
data := []string{"2010-01-02T12:32:00Z", "2010-04-20T12:32:00Z", "2011-04-20T12:32:00Z"}
|
||||
|
|
@ -7015,19 +7016,90 @@ func TestMissingKeyRegression(t *testing.T) {
|
|||
// (single and multi-node clusters, different endpoints for the
|
||||
// queries (HTTP, GRPC, Postgres), etc.).
|
||||
func TestVariousQueries(t *testing.T) {
|
||||
for _, clusterSize := range []int{1, 3, 4, 7} {
|
||||
for _, clusterSize := range []int{1, 3, 5} {
|
||||
clusterSize := clusterSize
|
||||
t.Run(fmt.Sprintf("%d-node", clusterSize), func(t *testing.T) {
|
||||
c := test.MustRunCluster(t, clusterSize)
|
||||
defer c.Close()
|
||||
|
||||
// put a variety of data into the cluster
|
||||
populateTestData(t, c)
|
||||
backupTest(t, c, usersIndex)
|
||||
|
||||
variousQueries(t, c)
|
||||
variousQueriesOnTimeFields(t, c)
|
||||
variousQueriesOnPercentiles(t, c)
|
||||
variousQueriesCountDistinctTimestamp(t, c)
|
||||
backupTest(t, c, "") // test backup/restore of all indexes
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func backupTest(t *testing.T, c *test.Cluster, index string) {
|
||||
// should this really be in executor? No. But all these
|
||||
// integration-y query tests probably shouldn't be either. My goal
|
||||
// putting this here is to take advantage of already-existing
|
||||
// clusters and data.
|
||||
|
||||
sum := chkSumCluster(t, c)
|
||||
|
||||
backupDir := backupCluster(t, c, index)
|
||||
|
||||
cnew := test.MustRunCluster(t, 3) // this way we test 1->3 3->3 5->3
|
||||
defer cnew.Close()
|
||||
|
||||
restoreCluster(t, backupDir, cnew)
|
||||
|
||||
sumNew := chkSumCluster(t, cnew)
|
||||
|
||||
if sum != sumNew {
|
||||
t.Fatalf("old/new checksum mismatch, old:\n%s\nnew:\n%s", sum, sumNew)
|
||||
}
|
||||
}
|
||||
|
||||
func chkSumCluster(t *testing.T, c *test.Cluster) string {
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
chkSum := ctl.NewChkSumCommand(nil, buf, buf)
|
||||
chkSum.Host = c.Nodes[len(c.Nodes)-1].URL()
|
||||
if err := chkSum.Run(context.Background()); err != nil {
|
||||
t.Fatalf("running checksum: %v", err)
|
||||
}
|
||||
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
func backupCluster(t *testing.T, c *test.Cluster, index string) (backupDir string) {
|
||||
td, err := testhook.TempDir(t, "backupTest")
|
||||
if err != nil {
|
||||
t.Fatalf("can't even get a temp dir, what a ripoff: %v", err)
|
||||
}
|
||||
td = td + "/backupTest"
|
||||
|
||||
buf := &bytes.Buffer{}
|
||||
backupCommand := ctl.NewBackupCommand(nil, buf, buf)
|
||||
backupCommand.Host = c.Nodes[len(c.Nodes)-1].URL() // don't pick node 0 so we don't always get primary (better code coverage)
|
||||
backupCommand.Index = index
|
||||
backupCommand.OutputDir = td
|
||||
|
||||
if err := backupCommand.Run(context.Background()); err != nil {
|
||||
t.Log(buf.String())
|
||||
t.Fatalf("running backup: %v", err)
|
||||
}
|
||||
return td
|
||||
}
|
||||
|
||||
func restoreCluster(t *testing.T, backupDir string, c *test.Cluster) {
|
||||
buf := &bytes.Buffer{}
|
||||
|
||||
restore := ctl.NewRestoreCommand(nil, buf, buf)
|
||||
restore.Host = c.Nodes[len(c.Nodes)-1].URL()
|
||||
restore.Path = backupDir
|
||||
if err := restore.Run(context.Background()); err != nil {
|
||||
t.Fatalf("restoring: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// tests for abbreviating time values in queries
|
||||
func variousQueriesOnPercentiles(t *testing.T, c *test.Cluster) {
|
||||
// todo, make rand more random, 42 isnt the answer to everything
|
||||
|
|
@ -7332,10 +7404,12 @@ func variousQueriesOnTimeFields(t *testing.T, c *test.Cluster) {
|
|||
}
|
||||
}
|
||||
|
||||
func variousQueries(t *testing.T, c *test.Cluster) {
|
||||
var usersIndex = "users"
|
||||
|
||||
func populateTestData(t *testing.T, c *test.Cluster) {
|
||||
// Create and populate "likenums" similar to "likes", but without keys on the field.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums")
|
||||
c.ImportIDKey(t, "users", "likenums", []test.KeyID{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likenums")
|
||||
c.ImportIDKey(t, usersIndex, "likenums", []test.KeyID{
|
||||
{ID: 1, Key: "userA"},
|
||||
{ID: 2, Key: "userB"},
|
||||
{ID: 3, Key: "userC"},
|
||||
|
|
@ -7353,8 +7427,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "likes" field.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys())
|
||||
c.ImportKeyKey(t, "users", "likes", [][2]string{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "likes", pilosa.OptFieldKeys())
|
||||
c.ImportKeyKey(t, usersIndex, "likes", [][2]string{
|
||||
{"molecula", "userA"},
|
||||
{"pilosa", "userB"},
|
||||
{"pangolin", "userC"},
|
||||
|
|
@ -7370,8 +7444,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "dinner" field.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys())
|
||||
c.ImportKeyKey(t, "users", "dinner", [][2]string{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "dinner", pilosa.OptFieldKeys())
|
||||
c.ImportKeyKey(t, usersIndex, "dinner", [][2]string{
|
||||
{"leftovers", "userB"},
|
||||
{"pizza", "userA"},
|
||||
{"pizza", "userB"},
|
||||
|
|
@ -7381,11 +7455,11 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "places_visited" time field.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM")))
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "places_visited", pilosa.OptFieldKeys(), pilosa.OptFieldTypeTime(pilosa.TimeQuantum("YM")))
|
||||
ts2019Jan01 := int64(1546300800) * 1e+9 // 2019 January 1st 0:00:00
|
||||
ts2019Aug01 := int64(1564617600) * 1e+9 // 2019 August 1st 0:00:00
|
||||
ts2020Jan01 := int64(1577836800) * 1e+9 // 2020 January 1st 0:00:00
|
||||
c.ImportTimeQuantumKey(t, "users", "places_visited", []test.TimeQuantumKey{
|
||||
c.ImportTimeQuantumKey(t, usersIndex, "places_visited", []test.TimeQuantumKey{
|
||||
// 2019 January: nairobi, paris, austin, toronto
|
||||
{RowKey: "nairobi", ColKey: "userB", Ts: ts2019Jan01},
|
||||
{RowKey: "paris", ColKey: "userC", Ts: ts2019Jan01},
|
||||
|
|
@ -7405,8 +7479,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "affinity" int field with negative, positive, zero and null values.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000))
|
||||
c.ImportIntKey(t, "users", "affinity", []test.IntKey{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "affinity", pilosa.OptFieldTypeInt(-1000, 1000))
|
||||
c.ImportIntKey(t, usersIndex, "affinity", []test.IntKey{
|
||||
{Val: 10, Key: "userA"},
|
||||
{Val: -10, Key: "userB"},
|
||||
{Val: 5, Key: "userC"},
|
||||
|
|
@ -7415,8 +7489,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
})
|
||||
|
||||
// Create and populate "net_worth" int field with positive values.
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000))
|
||||
c.ImportIntKey(t, "users", "net_worth", []test.IntKey{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "net_worth", pilosa.OptFieldTypeInt(-100000000, 100000000))
|
||||
c.ImportIntKey(t, usersIndex, "net_worth", []test.IntKey{
|
||||
{Val: 1, Key: "userA"},
|
||||
{Val: 10, Key: "userB"},
|
||||
{Val: 100, Key: "userC"},
|
||||
|
|
@ -7425,8 +7499,8 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
{Val: 100000, Key: "userF"},
|
||||
})
|
||||
|
||||
c.CreateField(t, "users", pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000))
|
||||
c.ImportIntKey(t, "users", "zip_code", []test.IntKey{
|
||||
c.CreateField(t, usersIndex, pilosa.IndexOptions{Keys: true, TrackExistence: true}, "zip_code", pilosa.OptFieldTypeInt(0, 100000))
|
||||
c.ImportIntKey(t, usersIndex, "zip_code", []test.IntKey{
|
||||
{Val: 78739, Key: "userA"},
|
||||
{Val: 78739, Key: "userB"},
|
||||
{Val: 19707, Key: "userC"},
|
||||
|
|
@ -7434,7 +7508,12 @@ func variousQueries(t *testing.T, c *test.Cluster) {
|
|||
{Val: 86753, Key: "userE"},
|
||||
{Val: 78739, Key: "userG"},
|
||||
})
|
||||
}
|
||||
|
||||
func variousQueries(t *testing.T, c *test.Cluster) {
|
||||
// NOTE: this relies on populateTestData being called first
|
||||
|
||||
// define and run a bunch of tests
|
||||
tests := []struct {
|
||||
query string
|
||||
qrVerifier func(t *testing.T, resp pilosa.QueryResponse)
|
||||
|
|
@ -7781,8 +7860,8 @@ leftovers,1
|
|||
|
||||
for i, tst := range tests {
|
||||
t.Run(fmt.Sprintf("%d-%s", i, tst.query), func(t *testing.T) {
|
||||
resp := c.Query(t, "users", tst.query)
|
||||
tr := c.QueryGRPC(t, "users", tst.query)
|
||||
resp := c.Query(t, usersIndex, tst.query)
|
||||
tr := c.QueryGRPC(t, usersIndex, tst.query)
|
||||
if tst.qrVerifier != nil {
|
||||
tst.qrVerifier(t, resp)
|
||||
}
|
||||
|
|
@ -8352,7 +8431,7 @@ func MinMaxTimestampNodeTester(t *testing.T, numNodes int) {
|
|||
defer c.Close()
|
||||
|
||||
// create an index and timestamp field
|
||||
c.CreateField(t, index, pilosa.IndexOptions{}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s"))
|
||||
c.CreateField(t, index, pilosa.IndexOptions{TrackExistence: true}, field, pilosa.OptFieldTypeTimestamp(time.Unix(0, 0), "s"))
|
||||
|
||||
// add some data
|
||||
expected := "2010-01-02T12:32:00Z"
|
||||
|
|
|
|||
4
field.go
4
field.go
|
|
@ -1884,7 +1884,7 @@ func applyDefaultOptions(o *FieldOptions) FieldOptions {
|
|||
// are included.
|
||||
func (o *FieldOptions) MarshalJSON() ([]byte, error) {
|
||||
switch o.Type {
|
||||
case FieldTypeSet:
|
||||
case FieldTypeSet, "":
|
||||
return json.Marshal(struct {
|
||||
Type string `json:"type"`
|
||||
CacheType string `json:"cacheType"`
|
||||
|
|
@ -1975,7 +1975,7 @@ func (o *FieldOptions) MarshalJSON() ([]byte, error) {
|
|||
o.Type,
|
||||
})
|
||||
}
|
||||
return nil, errors.New("invalid field type")
|
||||
return nil, errors.Errorf("invalid field type: '%s'", o.Type)
|
||||
}
|
||||
|
||||
// MinTimestamp returns the minimum value for a timestamp field.
|
||||
|
|
|
|||
|
|
@ -2326,6 +2326,12 @@ func (h *Handler) handleGetTranslateData(w http.ResponseWriter, r *http.Request)
|
|||
|
||||
// Retrieve partition data from holder.
|
||||
p, err := h.api.TranslateData(r.Context(), q.Get("index"), int(partition))
|
||||
if redir, ok := err.(pilosa.RedirectError); ok {
|
||||
newURL := *r.URL
|
||||
newURL.Host = redir.HostPort
|
||||
http.Redirect(w, r, newURL.String(), http.StatusSeeOther)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusNotFound)
|
||||
return
|
||||
|
|
|
|||
|
|
@ -1,57 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
declare STATUS="NORMAL"
|
||||
declare TIMEOUT=30
|
||||
sleep 4
|
||||
STATUS=$STATUS timeout -s TERM $TIMEOUT bash -c \
|
||||
'while [[ ${STATUS_RECEIVED} != ${STATUS} ]];\
|
||||
do STATUS_RECEIVED=$(curl --connect-timeout 1 -s pilosa0:10101/status | jq -r ".state") && \
|
||||
echo "received status: $STATUS_RECEIVED" && \
|
||||
sleep 1;\
|
||||
done;'
|
||||
echo "NOW DO STUFF"
|
||||
datagen --source kitchensink_keyed -e 9999 --pilosa.index sink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101
|
||||
before=$(/featurebase chksum --host pilosa0:10101)
|
||||
/featurebase backup -o backupdir --host pilosa0:10101
|
||||
curl -X DELETE -s pilosa0:10101/index/sink
|
||||
/featurebase restore -s backupdir --host pilosa0:10101
|
||||
after=$(/featurebase chksum --host pilosa0:10101)
|
||||
if [ "$before" = "$after" ]; then
|
||||
echo "PASS Cluster"
|
||||
else
|
||||
echo "FAIL Single"
|
||||
exit 1
|
||||
fi
|
||||
/featurebase restore -s backupdir --host pilosax:10101
|
||||
single=$(/featurebase chksum --host pilosax:10101)
|
||||
if [ "$before" = "$single" ]; then
|
||||
echo "PASS Single"
|
||||
exit 0
|
||||
else
|
||||
echo "FAIL Single"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
datagen --source texas_health -e 9999 --pilosa.index newsink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101
|
||||
before=$(/featurebase chksum --host pilosa0:10101)
|
||||
/featurebase backup -o newbackupdir --host pilosa0:10101 --index newsink
|
||||
curl -X DELETE -s pilosa0:10101/index/newsink
|
||||
/featurebase restore -s newbackupdir --host pilosa0:10101
|
||||
after=$(/featurebase chksum --host pilosa0:10101)
|
||||
if [ "$before" = "$after" ]; then
|
||||
echo "PASS Cluster Table"
|
||||
else
|
||||
echo "FAIL Single Table"
|
||||
exit 1
|
||||
fi
|
||||
/featurebase restore -s newbackupdir --host pilosax:10101
|
||||
single=$(/featurebase chksum --host pilosax:10101)
|
||||
if [ "$before" = "$single" ]; then
|
||||
echo "PASS Single Table"
|
||||
exit 0
|
||||
else
|
||||
echo "FAIL Single Table"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -568,6 +568,8 @@ func (s *InMemTranslateStore) WriteTo(w io.Writer) (int64, error) {
|
|||
// don't expect to use InMemTranslateStore much, it's mostly there to
|
||||
// avoid disk load during testing.
|
||||
func (s *InMemTranslateStore) ReadFrom(r io.Reader) (count int64, err error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var bytes []byte
|
||||
bytes, err = ioutil.ReadAll(r)
|
||||
count = int64(len(bytes))
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue