diff --git a/.circleci/config.yml b/.circleci/config.yml index 0ae2f1f43..30b2ef5e4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -35,7 +35,7 @@ commands: - checkout - restore-mod-cache skip-if-root-unchanged: - description: "skips the parent job if the PR includes no changes to pilosa" + description: "skips the parent job if the PR includes no changes to featurebase" steps: - run: | ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true @@ -202,7 +202,7 @@ jobs: version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker GO_VERSION=1.15.8 - - run: docker run pilosa:$(git describe --tags) help + - run: docker run featurebase:$(git describe --tags) help dockerhub-upload-unstable: executor: name: golang @@ -212,8 +212,8 @@ jobs: version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker - - run: docker run pilosa:$(git describe --tags) help - - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >> + - run: docker run featurebase:$(git describe --tags) help + - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >> dockerhub-upload-stable: executor: name: golang @@ -223,9 +223,9 @@ jobs: version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711 - run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin - run: make docker - - run: docker run pilosa:$(git describe --tags) help - - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >> - - run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest + - run: docker run featurebase:$(git describe --tags) help + - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >> + - run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest workflows: build: diff --git a/Dockerfile b/Dockerfile index 20170e420..115855034 100644 --- a/Dockerfile +++ b/Dockerfile @@ -28,7 +28,7 @@ COPY . ./ COPY --from=lattice-builder /lattice/build /lattice RUN /go/bin/statik -src=/lattice -dest=/pilosa -RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS} +RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS} ##################### ### Pilosa runner ### @@ -40,7 +40,7 @@ LABEL maintainer "dev@molecula.com" RUN apk add --no-cache curl jq -COPY --from=pilosa-builder /pilosa/build/pilosa / +COPY --from=pilosa-builder /pilosa/build/featurebase / COPY LICENSE /LICENSE COPY NOTICE /NOTICE @@ -52,5 +52,5 @@ ENV PILOSA_DATA_DIR /data ENV PILOSA_BIND 0.0.0.0:10101 ENV PILOSA_BIND_GRPC 0.0.0.0:20101 -ENTRYPOINT ["/pilosa"] +ENTRYPOINT ["/featurebase"] CMD ["server"] diff --git a/Dockerfile-clustertests b/Dockerfile-clustertests index 66fe174e9..8c60f11bf 100644 --- a/Dockerfile-clustertests +++ b/Dockerfile-clustertests @@ -1,7 +1,7 @@ # This Dockerfile is used for cluster testing - it produces a much larger image # and includes all of Go as well as some utilities. -FROM golang:1.13 +FROM golang:1.16 LABEL maintainer "dev@pilosa.com" @@ -14,6 +14,10 @@ RUN cd /go/src/github.com/molecula/featurebase \ ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba RUN chmod +x /pumba +# add docker client to pause/unpause nodes +RUN apt update +RUN apt install -y docker.io + RUN cp /go/bin/featurebase /featurebase COPY LICENSE /LICENSE diff --git a/Dockerfile.pilosa b/Dockerfile.pilosa index 068279d14..a24c70ec7 100644 --- a/Dockerfile.pilosa +++ b/Dockerfile.pilosa @@ -10,7 +10,7 @@ WORKDIR /pilosa COPY . ./ -RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS} +RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS} ##################### ### Pilosa runner ### @@ -22,7 +22,7 @@ LABEL maintainer "dev@molecula.com" RUN apk add --no-cache curl jq -COPY --from=pilosa-builder /pilosa/build/pilosa / +COPY --from=pilosa-builder /pilosa/build/featurebase / COPY LICENSE /LICENSE COPY NOTICE /NOTICE @@ -34,5 +34,5 @@ ENV PILOSA_DATA_DIR /data ENV PILOSA_BIND 0.0.0.0:10101 ENV PILOSA_BIND_GRPC 0.0.0.0:20101 -ENTRYPOINT ["/pilosa"] +ENTRYPOINT ["/featurebase"] CMD ["server"] diff --git a/Dockerfile.runner b/Dockerfile.runner index 3b1be7fd9..12f49e13c 100644 --- a/Dockerfile.runner +++ b/Dockerfile.runner @@ -10,7 +10,7 @@ WORKDIR /pilosa COPY . ./ -RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS} +RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS} FROM moleculacorp/idk as idk LABEL maintainer "dev@molecula.com" @@ -18,6 +18,6 @@ RUN apt-get update -y RUN apt-get install -y bash curl jq -COPY --from=pilosa-builder /pilosa/build/pilosa / +COPY --from=pilosa-builder /pilosa/build/featurebase / COPY testBackupRestore.sh / CMD ["bash","/testBackupRestore.sh"] diff --git a/Makefile b/Makefile index 6fb679f91..130cae3ce 100644 --- a/Makefile +++ b/Makefile @@ -116,6 +116,7 @@ release: check-clean generate-statik-docker $(MAKE) release-build GOOS=darwin GOARCH=amd64 $(MAKE) release-build GOOS=darwin GOARCH=arm64 $(MAKE) release-build GOOS=linux GOARCH=amd64 + $(MAKE) release-build GOOS=linux GOARCH=arm64 # Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI. release-sans-ui: check-clean @@ -123,6 +124,7 @@ release-sans-ui: check-clean $(MAKE) release-build GOOS=darwin GOARCH=amd64 $(MAKE) release-build GOOS=darwin GOARCH=arm64 $(MAKE) release-build GOOS=linux GOARCH=amd64 + $(MAKE) release-build GOOS=linux GOARCH=arm64 # try (e.g.) internal/clustertests/docker-compose-replication2.yml DOCKER_COMPOSE=internal/clustertests/docker-compose.yml @@ -133,10 +135,9 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml # pilosa image. clustertests: vendor docker-compose -f $(DOCKER_COMPOSE) down - docker-compose -f $(DOCKER_COMPOSE) build client1 + docker-compose -f $(DOCKER_COMPOSE) build docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1 - # Like clustertests, but rebuilds all images. clustertests-build: vendor docker-compose -f $(DOCKER_COMPOSE) down -v @@ -197,6 +198,7 @@ generate: generate-protoc generate-statik generate-stringer generate-pql # Create release using Docker docker-release: $(MAKE) docker-build GOOS=linux GOARCH=amd64 + $(MAKE) docker-build GOOS=linux GOARCH=arm64 $(MAKE) docker-build GOOS=darwin GOARCH=amd64 $(MAKE) docker-build GOOS=darwin GOARCH=arm64 @@ -206,28 +208,28 @@ docker-build: vendor --build-arg GO_VERSION=$(GO_VERSION) \ --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \ --target pilosa-builder \ - --tag pilosa:build . - docker create --name pilosa-build pilosa:build - mkdir -p build/pilosa-$(VERSION_ID) - docker cp pilosa-build:/pilosa/build/. ./build/pilosa-$(VERSION_ID) - cp NOTICE LICENSE ./build/pilosa-$(VERSION_ID) - docker rm pilosa-build - tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/ + --tag featurebase:build . + docker create --name featurebase-build featurebase:build + mkdir -p build/featurebase-$(VERSION_ID) + docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID) + cp NOTICE LICENSE ./build/featurebase-$(VERSION_ID) + docker rm featurebase-build + tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/ # Create Docker image from Dockerfile docker-image: vendor docker build \ --build-arg GO_VERSION=$(GO_VERSION) \ --build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \ - --tag pilosa:$(VERSION) . - @echo Created docker image: pilosa:$(VERSION) + --tag featurebase:$(VERSION) . + @echo Created docker image: featurebase:$(VERSION) # Create docker image (alias) docker: docker-image # alias # Tag and push a Docker image docker-tag-push: vendor - docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET) + docker tag "featurebase:$(VERSION)" $(DOCKER_TARGET) docker push $(DOCKER_TARGET) @echo Pushed docker image: $(DOCKER_TARGET) diff --git a/api.go b/api.go index fe5723365..ca3506a0e 100644 --- a/api.go +++ b/api.go @@ -36,6 +36,8 @@ import ( "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/ingest" + + //"github.com/molecula/featurebase/v2/pg" "github.com/molecula/featurebase/v2/pql" "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/stats" @@ -1824,6 +1826,44 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu return nil } +// ingestNodeOperationsForFields does the actual work of applying operations +// to a given index with a map of known fields and an already-parsed +// ShardedRequest. This is used locally on the node that first receives +// the request, after it does the parsing, and on other nodes because the +// format they get is already that rather than JSON, so it's the common +// path *after* key translation and sorting into shards. +func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error { + eg, ctx := errgroup.WithContext(ctx) + for shard, ops := range req.Ops { + // create new local copies of these values so the goroutine uses these + // copies, and doesn't read the actual loop variables, which are being + // changed by the loop. + shard, ops := shard, ops + eg.Go(func() error { + return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) + }) + } + return eg.Wait() +} + +// IngestNodeOperations handles protobuf-formatted data which does not need +// key translation and is applicable to this specific node. +func (api *API) IngestNodeOperations(ctx context.Context, qcx *Qcx, indexName string, req *ingest.ShardedRequest) error { + index := api.holder.Index(indexName) + if index == nil { + api.server.logger.Errorf("ingest: no such index %q", indexName) + return newNotFoundError(ErrIndexNotFound, indexName) + } + fields := index.Fields() + knownFields := map[string]*Field{} + for _, field := range fields { + knownFields[field.name] = field + } + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, req) +} + +// IngestOperations handles JSON-formatted data which may need key translation +// and may be for any or all nodes. func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string, stream io.Reader) error { span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations") defer span.Finish() @@ -1839,34 +1879,32 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string return newNotFoundError(ErrIndexNotFound, indexName) } fields := index.Fields() - var lookup ingest.KeyLookupFunc + var indexKeys ingest.KeyTranslator if index.Keys() { - lookup = func(keys ...string) (map[string]uint64, error) { - return api.cluster.createIndexKeys(ctx, indexName, keys...) - } + indexKeys = newIngestKeyTranslatorFromCluster(ctx, api.cluster, indexName) } - codec, err := ingest.NewJSONCodec(lookup) + codec, err := ingest.NewJSONCodec(indexKeys) if err != nil { return errors.Wrap(err, "creating JSON codec") } knownFields := map[string]*Field{} for _, field := range fields { - var lookup ingest.KeyLookupFunc + var keys ingest.KeyTranslator if field.usesKeys { - lookup = field.translateStore.CreateKeys + keys = newIngestKeyTranslatorFromStore(field.translateStore) } knownFields[field.name] = field switch field.Type() { case "set": - if err = codec.AddSetField(field.name, lookup); err != nil { + if err = codec.AddSetField(field.name, keys); err != nil { return fmt.Errorf("adding set field to codec: %w", err) } case "time": - if err = codec.AddTimeQuantumField(field.name, lookup); err != nil { + if err = codec.AddTimeQuantumField(field.name, keys); err != nil { return fmt.Errorf("adding time quantum field to codec: %w", err) } case "mutex": - if err = codec.AddMutexField(field.name, lookup); err != nil { + if err = codec.AddMutexField(field.name, keys); err != nil { return fmt.Errorf("adding mutex field to codec: %w", err) } case "bool": @@ -1874,7 +1912,7 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string return fmt.Errorf("adding bool field to codec: %w", err) } case "int": - if err = codec.AddIntField(field.name, lookup); err != nil { + if err = codec.AddIntField(field.name, keys); err != nil { return fmt.Errorf("adding int field to codec: %w", err) } case "decimal": @@ -1894,17 +1932,53 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string if err != nil { return errors.Wrap(err, "parsing input data") } - sharded, err := req.ByShard() + sharded, err := codec.RequestByShard(req) if err != nil { return errors.Wrap(err, "sharding input data") } - eg, ctx := errgroup.WithContext(ctx) + // now that we have this, let's assign the shards to nodes + snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN) + // oh hey an easy case: we're presumably the only node + if len(snap.Nodes) == 1 { + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) + } + // Created new ShardedRequest objects for every node, giving each of them + // all the shards that apply to them. + byNode := make(map[string]*ingest.ShardedRequest) for shard, ops := range sharded.Ops { - // loop variable shadow capture is the go equivalent of man door hook hand - shard, ops := shard, ops - eg.Go(func() error { - return api.applyOperations(ctx, qcx, index, shard, knownFields, ops) - }) + nodes := snap.ShardNodes(indexName, shard) + for _, node := range nodes { + forThisShard := byNode[node.ID] + if forThisShard == nil { + // Create new ShardedRequest for the target node, with its op map + // mapping this shard to the ops for this shard. + byNode[node.ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}} + continue + } + // Add this shard to the existing ShardedRequest's Ops map. Note that + // we don't have to worry about overwrites; we can't have seen this + // shard before, because we're in a range loop on a map where the shard + // is the key. + forThisShard.Ops[shard] = ops + } + } + eg, ctx := errgroup.WithContext(ctx) + for _, node := range snap.Nodes { + node := node + sharded := byNode[node.ID] + // Sometimes, there's nothing for a specific node. + if sharded == nil { + continue + } + if node.ID == api.NodeID() { + eg.Go(func() error { + return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded) + }) + } else { + eg.Go(func() error { + return api.server.defaultClient.IngestNodeOperations(ctx, &node.URI, indexName, sharded) + }) + } } return eg.Wait() } @@ -2692,9 +2766,9 @@ func mergeIDLists(dst []uint64, src []uint64) []uint64 { return dst[i] < dst[j] }) // dedup. - n := 0 + n := 1 prev := dst[0] - for i := 0; i < len(dst); i++ { + for i := 1; i < len(dst); i++ { if dst[i] != prev { dst[n] = dst[i] n++ @@ -2712,9 +2786,9 @@ func mergeKeyLists(dst []string, src []string) []string { return dst[i] < dst[j] }) // dedup. - n := 0 + n := 1 prev := dst[0] - for i := 0; i < len(dst); i++ { + for i := 1; i < len(dst); i++ { if dst[i] != prev { dst[n] = dst[i] n++ @@ -3024,6 +3098,9 @@ processing: } return result, ctx.Err() } +func (api *API) Plan(ctx context.Context, q string) (*Stmt, error) { + return api.server.PlanSQL(ctx, q) +} type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` @@ -3085,6 +3162,7 @@ const ( apiIDReset apiPartitionNodes apiIngestOperations + apiIngestNodeOperations apiMutexCheck ) @@ -3154,5 +3232,6 @@ var methodsNormal = map[apiMethod]struct{}{ apiIDReset: {}, apiPartitionNodes: {}, apiIngestOperations: {}, + apiIngestNodeOperations: {}, apiMutexCheck: {}, } diff --git a/api_test.go b/api_test.go index e4826374f..50a59a03e 100644 --- a/api_test.go +++ b/api_test.go @@ -860,7 +860,13 @@ type mutexCheckField struct { } func TestAPI_MutexCheck(t *testing.T) { - c := test.MustRunCluster(t, 3) + c := test.MustNewCluster(t, 3) + for _, c := range c.Nodes { + c.Config.Cluster.ReplicaN = 2 + } + if err := c.Start(); err != nil { + t.Fatalf("starting cluster: %v", err) + } defer c.Close() m0 := c.GetNode(0) diff --git a/bolt.go b/bolt.go index e679b1400..12435abef 100644 --- a/bolt.go +++ b/bolt.go @@ -135,8 +135,12 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag if !DirExists(path) { PanicOn(os.MkdirAll(dir, 0755)) } + fsyncEnabled := true + if cfg != nil { + fsyncEnabled = cfg.FsyncEnabled + } - db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize}) + db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !fsyncEnabled}) if err != nil { return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path)) } @@ -187,6 +191,7 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag openTx: make(map[*BoltTx]bool), DeleteEmptyContainer: true, + fsyncEnabled: cfg.FsyncEnabled, } r.unprotectedRegister(w) @@ -226,7 +231,7 @@ func (w *BoltWrapper) CloseDB() error { func (w *BoltWrapper) OpenDB() error { w.muDb.Lock() defer w.muDb.Unlock() - db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize}) + db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !w.fsyncEnabled}) if err != nil { return err } @@ -315,6 +320,7 @@ type BoltWrapper struct { doAllocZero bool DeleteEmptyContainer bool + fsyncEnabled bool // for tracking whether our initial config wanted fsync on openTx map[*BoltTx]bool } diff --git a/bolt_test.go b/bolt_test.go index 5c83ae1af..1dc03b7d0 100644 --- a/bolt_test.go +++ b/bolt_test.go @@ -21,6 +21,7 @@ import ( "testing" "github.com/molecula/featurebase/v2/roaring" + "github.com/molecula/featurebase/v2/storage" . "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck ) @@ -88,7 +89,7 @@ func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) { var err error fn := path PanicOn(os.RemoveAll(fn)) - ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, nil) + ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, &storage.Config{FsyncEnabled: false}) PanicOn(err) w = ww.(*BoltWrapper) diff --git a/boltdb/translate.go b/boltdb/translate.go index 076c77f37..35ef2b08f 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -55,8 +55,8 @@ const ( ) // OpenTranslateStore opens and initializes a boltdb translation store. -func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) { - s := NewTranslateStore(index, field, partitionID, partitionN) +func OpenTranslateStore(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (pilosa.TranslateStore, error) { + s := NewTranslateStore(index, field, partitionID, partitionN, fsyncEnabled) s.Path = path if err := s.Open(); err != nil { return nil, err @@ -88,22 +88,24 @@ type TranslateStore struct { once sync.Once closing chan struct{} - readOnly bool - writeNotify chan struct{} + readOnly bool + fsyncEnabled bool + writeNotify chan struct{} // File path to database file. Path string } // NewTranslateStore returns a new instance of TranslateStore. -func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore { +func NewTranslateStore(index, field string, partitionID, partitionN int, fsyncEnabled bool) *TranslateStore { return &TranslateStore{ - index: index, - field: field, - partitionID: partitionID, - partitionN: partitionN, - closing: make(chan struct{}), - writeNotify: make(chan struct{}), + index: index, + field: field, + partitionID: partitionID, + partitionN: partitionN, + closing: make(chan struct{}), + writeNotify: make(chan struct{}), + fsyncEnabled: fsyncEnabled, } } @@ -120,7 +122,7 @@ func (s *TranslateStore) Open() (err error) { if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path)) - } else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil { + } else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !s.fsyncEnabled}); err != nil { return errors.Wrapf(err, "open file: %s", err) } diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index a42d68d50..61f2e61c3 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -393,7 +393,7 @@ func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore { panic(err) } - s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN) + s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN, false) s.Path = f.Name() return s } diff --git a/client.go b/client.go index 62f16e240..8726c440a 100644 --- a/client.go +++ b/client.go @@ -19,6 +19,7 @@ import ( "io" "time" + "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/topology" ) @@ -81,6 +82,7 @@ type InternalClient interface { ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) MutexCheck(ctx context.Context, uri *pnet.URI, index string, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error) + IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error IDAllocDataReader(ctx context.Context) (io.ReadCloser, error) IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error) @@ -218,6 +220,10 @@ func (n nopInternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, index, return nil, nil } +func (n nopInternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { + return nil +} + func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) { return nil, nil } diff --git a/cluster.go b/cluster.go index e86883c1c..2843b732f 100644 --- a/cluster.go +++ b/cluster.go @@ -24,6 +24,7 @@ import ( "time" "github.com/molecula/featurebase/v2/disco" + "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/roaring" "github.com/molecula/featurebase/v2/topology" @@ -1165,6 +1166,9 @@ func (c *cluster) followResizeInstruction(ctx context.Context, instr *ResizeInst } } + // fire off translation sync + _ = c.translationSyncer.Reset() + return nil } @@ -1179,6 +1183,8 @@ func (c *cluster) resizeAbort() error { if c.resizeCancel != nil { c.resizeCancel() } + // fire off translation sync + _ = c.translationSyncer.Reset() return nil } @@ -1561,6 +1567,38 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys return ids, nil } +// This implements ingest's key translator interface on a cluster/index pair. +type clusterKeyTranslator struct { + ctx context.Context // we're created within a request context and need to pass that to cluster ops + c *cluster + indexName string +} + +var _ ingest.KeyTranslator = &clusterKeyTranslator{} + +func (i clusterKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + return i.c.createIndexKeys(i.ctx, i.indexName, keys...) +} + +func (i clusterKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + keys, err := i.c.translateIndexIDs(i.ctx, i.indexName, ids) + if err != nil { + return nil, err + } + if len(keys) != len(ids) { + return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) + } + out := make(map[uint64]string, len(keys)) + for i, id := range ids { + out[id] = keys[i] + } + return out, nil +} + +func newIngestKeyTranslatorFromCluster(ctx context.Context, c *cluster, indexName string) *clusterKeyTranslator { + return &clusterKeyTranslator{ctx: ctx, c: c, indexName: indexName} +} + // TODO: remove this when it is no longer used func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) { keys := make([]string, 0, len(keySet)) diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 8af84efed..cc55ae9aa 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -155,7 +155,10 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index { if err != nil { panic(err) } - h := NewHolder(path, nil) + cfg := DefaultHolderConfig() + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false + h := NewHolder(path, cfg) PanicOn(h.Open()) index, err := h.CreateIndex(name, IndexOptions{}) testhook.Cleanup(tb, func() { diff --git a/cmd/backup.go b/cmd/backup.go index 68baeb014..d81029d7a 100644 --- a/cmd/backup.go +++ b/cmd/backup.go @@ -40,6 +40,7 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file. flags.BoolVar(&cmd.NoSync, "no-sync", false, "disable file sync") flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "number of concurrent backup goroutines") flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.") + flags.StringVar(&cmd.Index, "index", "", "index to backup, default backs up all indexes. ") ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification) return ccmd } diff --git a/cmd/roaring-migrate/ctim_darwin.go b/cmd/roaring-migrate/ctim_darwin.go new file mode 100644 index 000000000..0b50491ab --- /dev/null +++ b/cmd/roaring-migrate/ctim_darwin.go @@ -0,0 +1,27 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build darwin + +package main + +import ( + "syscall" +) + +func CTimeNano(stat *syscall.Stat_t) int64 { + NANOS := int64(1e9) // number of nanosecs in 1 sec + ts := stat.Ctimespec + return ts.Sec*NANOS + ts.Nsec +} diff --git a/ingest/translate.go b/cmd/roaring-migrate/ctim_linux.go similarity index 78% rename from ingest/translate.go rename to cmd/roaring-migrate/ctim_linux.go index 7ce7ea6dd..b9e5d70e1 100644 --- a/ingest/translate.go +++ b/cmd/roaring-migrate/ctim_linux.go @@ -1,4 +1,4 @@ -// Copyright 2021 Molecula Corp. +// Copyright 2017 Pilosa Corp. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,4 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ingest +// +build linux + +package main + +import ( + "syscall" +) + +func CTimeNano(stat *syscall.Stat_t) int64 { + return stat.Ctim.Nano() +} diff --git a/cmd/roaring-migrate/main.go b/cmd/roaring-migrate/main.go index 60ff5bf7c..8b5b62d6c 100644 --- a/cmd/roaring-migrate/main.go +++ b/cmd/roaring-migrate/main.go @@ -24,7 +24,7 @@ import ( "strings" "syscall" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/rbf" "github.com/molecula/featurebase/v2/rbf/cfg" "github.com/molecula/featurebase/v2/roaring" @@ -148,7 +148,7 @@ func BuildSchema(dataDir string) ([]byte, error) { l = &local{ Name: index, Fields: make([]*pilosa.FieldInfo, 0), - CreatedAt: uint64(stat.Ctim.Nano()), + CreatedAt: uint64(CTimeNano(stat)), Options: *io, } //index options @@ -158,7 +158,7 @@ func BuildSchema(dataDir string) ([]byte, error) { field := t[2] if field != "_exists" { - fi, err := pilosa.UnmarshalFieldOptions(field, stat.Ctim.Nano(), content) + fi, err := pilosa.UnmarshalFieldOptions(field, CTimeNano(stat), content) if err != nil { return err } diff --git a/ctl/backup.go b/ctl/backup.go index e68ce52d2..cdcfb1427 100644 --- a/ctl/backup.go +++ b/ctl/backup.go @@ -24,7 +24,7 @@ import ( "os" "path/filepath" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/server" "github.com/molecula/featurebase/v2/topology" @@ -38,6 +38,9 @@ type BackupCommand struct { // nolint: maligned // Destination host and port. Host string `json:"host"` + // Optional Index filter + Index string `json:"index"` + // Path to write the backup to. OutputDir string @@ -91,6 +94,19 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) { if err != nil { return fmt.Errorf("getting schema: %w", err) } + if cmd.Index != "" { + for _, idx := range indexes { + if idx.Name == cmd.Index { + indexes = make([]*pilosa.IndexInfo, 0) + indexes = append(indexes, idx) + break + } + } + if len(indexes) <= 0 { + return fmt.Errorf("Index not found to back up") + } + } + schema := &pilosa.Schema{Indexes: indexes} // Ensure output directory doesn't exist; then create output directory. diff --git a/ctl/restore.go b/ctl/restore.go index 26d72c788..87d4d3c87 100644 --- a/ctl/restore.go +++ b/ctl/restore.go @@ -17,6 +17,7 @@ package ctl import ( "context" "crypto/tls" + "encoding/json" "errors" "fmt" "io" @@ -26,7 +27,7 @@ import ( "strconv" "strings" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/server" "github.com/molecula/featurebase/v2/topology" "golang.org/x/sync/errgroup" @@ -102,7 +103,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) { } else if err := cmd.restoreIDAlloc(ctx, primary); err != nil { return fmt.Errorf("cannot restore idalloc: %w", err) } - if err := cmd.restoreShards(ctx); err != nil { return fmt.Errorf("cannot restore shards: %w", err) } else if err := cmd.restoreIndexTranslation(ctx); err != nil { @@ -128,11 +128,47 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology. } defer f.Close() - cmd.Logger().Printf("Load Schema") - url := primary.URI.Path("/schema") - - var client http.Client - _, err = client.Post(url, "application/json", f) + existingSchema, err := cmd.client.Schema(ctx) + if len(existingSchema) == 0 { + cmd.Logger().Printf("Load Schema") + url := primary.URI.Path("/schema") + var client http.Client + _, err = client.Post(url, "application/json", f) + } else { + schema := &pilosa.Schema{} + if err := json.NewDecoder(f).Decode(schema); err != nil { + if err != nil { + return err + } + } + exists := func(indexName string) bool { + for _, i := range existingSchema { + if i.Name == indexName { + return true + } + } + return false + } + logger := cmd.Logger() + //NOTE SHOULD ONLY BE ONE + for _, index := range schema.Indexes { + if exists(index.Name) { + return fmt.Errorf("Index Exists %v", index.Name) + } + logger.Printf("Create INDEX %v", index.Name) + err = cmd.client.CreateIndex(ctx, index.Name, index.Options) + if err != nil { + return err + } + for _, field := range index.Fields { + logger.Printf("Create Field %v", field.Name) + err = cmd.client.CreateFieldWithOptions(ctx, index.Name, field.Name, field.Options) + if err != nil { + return err + } + } + } + } return err } diff --git a/ctl/server.go b/ctl/server.go index 2aa617295..6c4405135 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -111,6 +111,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.DurationVar((*time.Duration)(&srv.Config.Postgres.WriteTimeout), "postgres.write-timeout", time.Duration(srv.Config.Postgres.WriteTimeout), "Timeout for writes on a postgres connection. (set 0 to disable)") flags.Uint32Var(&srv.Config.Postgres.MaxStartupSize, "postgres.max-startup-size", srv.Config.Postgres.MaxStartupSize, "Maximum acceptable size of a postgres startup packet, in bytes. (set 0 to disable)") flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)") + flags.Uint16Var(&srv.Config.Postgres.SqlVersion, "postgres.sql-version", srv.Config.Postgres.SqlVersion, "Molecula Sql Handling Version (default 1)") // Disk and Memory usage cache for ui/usage endpoint flags.Float64Var(&srv.Config.UsageDutyCycle, "usage-duty-cycle", srv.Config.UsageDutyCycle, "Sets the percentage of time that is spent recalculating the disk and memory usage cache. 100.0 for always-running, 0 disables the cache and the /ui/usage endpoint.") diff --git a/dbshard_internal_test.go b/dbshard_internal_test.go index 81d64436c..9482d1504 100644 --- a/dbshard_internal_test.go +++ b/dbshard_internal_test.go @@ -330,6 +330,7 @@ func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) { cfg := mustHolderConfig() cfg.StorageConfig.Backend = "rbf" + cfg.StorageConfig.FsyncEnabled = false holder := NewHolder(tmpdir, cfg) defer holder.Close() diff --git a/encoding/proto/proto.go b/encoding/proto/proto.go index 69a456904..c5672e563 100644 --- a/encoding/proto/proto.go +++ b/encoding/proto/proto.go @@ -21,6 +21,7 @@ import ( "github.com/gogo/protobuf/proto" "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" + "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/pb" "github.com/molecula/featurebase/v2/pql" @@ -324,7 +325,18 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error { } decodeResizeAbortMessage(msg, mt) return nil - + case *ingest.ShardedRequest: + msg := &pb.ShardedIngestRequest{} + err := proto.Unmarshal(buf, msg) + if err != nil { + return errors.Wrap(err, "unmarshalling ShardedRequest") + } + req, err := s.decodeShardedIngestRequest(msg) + if err != nil { + return err + } + *mt = *req + return nil default: panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m)) } @@ -398,6 +410,8 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message { return s.encodeResizeNodeMessage(mt) case *pilosa.ResizeAbortMessage: return s.encodeResizeAbortMessage(mt) + case *ingest.ShardedRequest: + return s.encodeShardedIngestRequest(mt) } return nil } @@ -931,6 +945,48 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *pb.Tr return &pb.TransactionStats{} } +func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.ShardedIngestRequest { + if req == nil || len(req.Ops) == 0 { + return &pb.ShardedIngestRequest{} + } + out := &pb.ShardedIngestRequest{Ops: make(map[uint64]*pb.ShardIngestOperations, len(req.Ops))} + for shard, ops := range req.Ops { + out.Ops[shard] = s.encodeShardIngestOperations(ops) + } + return out +} + +func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations { + out := &pb.ShardIngestOperations{} + for _, op := range ops { + if op == nil { + continue + } + out.Ops = append(out.Ops, s.encodeShardIngestOperation(op)) + } + return out +} + +func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIngestOperation { + out := &pb.ShardIngestOperation{ + OpType: op.OpType.String(), + ClearRecordIDs: op.ClearRecordIDs, + ClearFields: op.ClearFields, + FieldOps: make(map[string]*pb.FieldOperation, len(op.FieldOps)), + } + for k, v := range op.FieldOps { + if v == nil { + continue + } + out.FieldOps[k] = &pb.FieldOperation{ + RecordIDs: v.RecordIDs, + Values: v.Values, + Signed: v.Signed, + } + } + return out +} + func (s Serializer) decodeResizeInstruction(ri *pb.ResizeInstruction, m *pilosa.ResizeInstruction) { m.JobID = ri.JobID m.Node = &topology.Node{} @@ -1829,3 +1885,57 @@ func decodeResizeNodeMessage(pb *pb.ResizeNodeMessage, m *pilosa.ResizeNodeMessa func decodeResizeAbortMessage(pb *pb.ResizeAbortMessage, m *pilosa.ResizeAbortMessage) { } + +func (s Serializer) decodeShardedIngestRequest(req *pb.ShardedIngestRequest) (*ingest.ShardedRequest, error) { + if req == nil || len(req.Ops) == 0 { + return &ingest.ShardedRequest{}, nil + } + out := &ingest.ShardedRequest{Ops: make(map[uint64][]*ingest.Operation, len(req.Ops))} + for shard, ops := range req.Ops { + var err error + out.Ops[shard], err = s.decodeShardIngestOperations(ops) + if err != nil { + return nil, err + } + } + return out, nil +} + +func (s Serializer) decodeShardIngestOperations(ops *pb.ShardIngestOperations) ([]*ingest.Operation, error) { + out := []*ingest.Operation{} + if len(ops.Ops) == 0 { + return out, nil + } + for _, op := range ops.Ops { + if op == nil { + continue + } + decoded, err := s.decodeShardIngestOperation(op) + if err != nil { + return nil, err + } + out = append(out, decoded) + } + return out, nil +} + +func (s Serializer) decodeShardIngestOperation(op *pb.ShardIngestOperation) (*ingest.Operation, error) { + opType, err := ingest.ParseOpType(op.OpType) + if err != nil { + return nil, err + } + out := &ingest.Operation{ + OpType: opType, + ClearRecordIDs: op.ClearRecordIDs, + ClearFields: op.ClearFields, + FieldOps: make(map[string]*ingest.FieldOperation, len(op.FieldOps)), + } + for k, v := range op.FieldOps { + out.FieldOps[k] = &ingest.FieldOperation{ + RecordIDs: v.RecordIDs, + Values: v.Values, + Signed: v.Signed, + } + } + return out, nil +} diff --git a/encoding/proto/proto_test.go b/encoding/proto/proto_test.go new file mode 100644 index 000000000..7ea635881 --- /dev/null +++ b/encoding/proto/proto_test.go @@ -0,0 +1,149 @@ +// Copyright 2021 Molecula Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package proto + +import ( + "errors" + "reflect" + "testing" + + "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/ingest" +) + +func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, expectedMarshalErr error, expectedUnmarshalErr error, expectedMismatchErr error) { + repr, err := s.Marshal(obj) + if err != nil { + if expectedMarshalErr == nil { + t.Fatalf("unexpected marshalling error %q", err.Error()) + } + if err.Error() != expectedMarshalErr.Error() { + t.Fatalf("expecting marshalling error %q, got %q", expectedMarshalErr.Error(), err.Error()) + } + } else { + if expectedMarshalErr != nil { + t.Fatalf("expected marshalling error %q, got no error", expectedMarshalErr.Error()) + } + } + + obj2 := reflect.New(reflect.TypeOf(obj).Elem()).Interface() + err = s.Unmarshal(repr, obj2) + if err != nil { + if expectedUnmarshalErr == nil { + t.Fatalf("unexpected unmarshalling error %q", err.Error()) + } + if err.Error() != expectedUnmarshalErr.Error() { + t.Fatalf("expecting unmarshalling error %q, got %q", expectedUnmarshalErr.Error(), err.Error()) + } + } else { + if expectedUnmarshalErr != nil { + t.Fatalf("expected unmarshalling error %q, got no error", expectedUnmarshalErr.Error()) + } + } + switch real := obj.(type) { + case *ingest.ShardedRequest: + real2 := obj2.(*ingest.ShardedRequest) + err := real.Compare(real2) + if err != nil { + if expectedMismatchErr == nil { + t.Fatalf("unexpected compare error %q", err.Error()) + } + if err.Error() != expectedMismatchErr.Error() { + t.Fatalf("expecting compare error %q, got %q", expectedMismatchErr.Error(), err.Error()) + } + } else { + if expectedMismatchErr != nil { + t.Fatalf("expected compare error %q, got no error", expectedMismatchErr.Error()) + } + } + default: + if !reflect.DeepEqual(obj, obj2) { + t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2) + } + } +} + +type shardedIngestRequestTest struct { + req *ingest.ShardedRequest + err error +} + +var shardedIngestRequestTestcases = []shardedIngestRequestTest{ + { + req: &ingest.ShardedRequest{ + Ops: map[uint64][]*ingest.Operation{ + 1: { + { + OpType: ingest.OpWrite, + ClearFields: []string{"clearField", "clearField2"}, + ClearRecordIDs: []uint64{1, 7, 9}, + FieldOps: map[string]*ingest.FieldOperation{ + "writeAll": { + RecordIDs: []uint64{3, 6, 8}, + Values: []uint64{0, 17, 34}, + Signed: []int64{-9, 23, 17}, + }, + "writeValues": { + RecordIDs: []uint64{3, 6, 8}, + Values: []uint64{0, 17, 34}, + }, + "writeSigned": { + RecordIDs: []uint64{3, 6, 8}, + Signed: []int64{-9, 23, 17}, + }, + }, + }, + { + OpType: ingest.OpSet, + FieldOps: map[string]*ingest.FieldOperation{ + "foo": nil, + }, + }, + }, + 2: {}, + 3: nil, + }, + }, + }, + { + req: &ingest.ShardedRequest{ + Ops: map[uint64][]*ingest.Operation{ + 1: { + { + OpType: ingest.OpWrite, + FieldOps: map[string]*ingest.FieldOperation{ + "writeAll": {}, + }, + }, + { + OpType: ingest.OpSet, + FieldOps: map[string]*ingest.FieldOperation{ + "foo": nil, + }, + }, + nil, + }, + }, + }, + err: errors.New("shard 1: expected 3 ops, got 2"), + }, +} + +func TestIngestRoundTrip(t *testing.T) { + for _, tc := range shardedIngestRequestTestcases { + t.Logf("next case") + testOneRoundTrip(t, DefaultSerializer, tc.req, nil, nil, tc.err) + } +} diff --git a/etcd/embed.go b/etcd/embed.go index 67711d4c1..7da2c9806 100644 --- a/etcd/embed.go +++ b/etcd/embed.go @@ -63,6 +63,7 @@ type Options struct { LClientSocket []*net.TCPListener BootstrapTimeout time.Duration + UnsafeNoFsync bool `toml:"no-fsync"` } var ( @@ -228,6 +229,7 @@ func parseOptions(opt Options) *embed.Config { cfg.InitialClusterToken = opt.ClusterName cfg.BootstrapTimeout = opt.BootstrapTimeout cfg.LCUrls = types.MustNewURLs([]string{opt.LClientURL}) + cfg.UnsafeNoFsync = opt.UnsafeNoFsync if opt.AClientURL != "" { cfg.ACUrls = types.MustNewURLs([]string{opt.AClientURL}) } else { diff --git a/executor.go b/executor.go index 69d96928f..ac9f4455e 100644 --- a/executor.go +++ b/executor.go @@ -1825,10 +1825,14 @@ func (e *executor) executeSumCountShard(ctx context.Context, qcx *Qcx, index str if err != nil { return ValCount{}, errors.Wrap(err, "computing sum") } - return ValCount{ + out := ValCount{ Val: int64(vsum) + (int64(vcount) * bsig.Base), Count: int64(vcount), - }, nil + } + if field.Type() == FieldTypeDecimal { + out.FloatVal = float64(int64(vsum)+(int64(vcount)*bsig.Base)) / math.Pow(10, float64(bsig.Scale)) + } + return out, nil } // executeMinShard calculates the min for bsiGroups on a shard. @@ -3044,6 +3048,13 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c aggType = "aggregate" } } + for _, res := range results { + if res.DecimalAgg != 0 && aggType == "sum" { + aggType = "decimalSum" + break + } + } + return NewGroupCounts(aggType, results...), nil } @@ -3132,9 +3143,10 @@ func (fr FieldRow) String() string { type aggregateType int const ( - nilAggregate aggregateType = 0 - sumAggregate aggregateType = 1 - distinctAggregate aggregateType = 2 + nilAggregate aggregateType = 0 + sumAggregate aggregateType = 1 + distinctAggregate aggregateType = 2 + decimalSumAggregate aggregateType = 3 ) // GroupCounts is a list of GroupCount. @@ -3152,6 +3164,8 @@ func (g *GroupCounts) AggregateColumn() string { return "sum" case distinctAggregate: return "aggregate" + case decimalSumAggregate: + return "decimalSum" default: return "" } @@ -3176,6 +3190,8 @@ func NewGroupCounts(agg string, groups ...GroupCount) *GroupCounts { aggType = sumAggregate case "aggregate": aggType = distinctAggregate + case "decimalSum": + aggType = decimalSumAggregate case "": aggType = nilAggregate default: @@ -3246,42 +3262,57 @@ func (g *GroupCounts) MarshalJSON() ([]byte, error) { if len(groups) == 0 { return []byte("[]"), nil } + switch g.aggregateType { case sumAggregate: counts = *(*[]groupCountSum)(unsafe.Pointer(&groups)) case distinctAggregate: counts = *(*[]groupCountAggregate)(unsafe.Pointer(&groups)) + case decimalSumAggregate: + counts = *(*[]groupCountDecimalSum)(unsafe.Pointer(&groups)) } return json.Marshal(counts) } // GroupCount represents a result item for a group by query. type GroupCount struct { - Group []FieldRow `json:"group"` - Count uint64 `json:"count"` - Agg int64 `json:"-"` + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` + Agg int64 `json:"-"` + DecimalAgg float64 `json:"-"` } type groupCountSum struct { - Group []FieldRow `json:"group"` - Count uint64 `json:"count"` - Agg int64 `json:"sum"` + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` + Agg int64 `json:"sum"` + DecimalAgg float64 `json:"-"` } type groupCountAggregate struct { - Group []FieldRow `json:"group"` - Count uint64 `json:"count"` - Agg int64 `json:"aggregate"` + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` + Agg int64 `json:"aggregate"` + DecimalAgg float64 `json:"-"` +} + +type groupCountDecimalSum struct { + Group []FieldRow `json:"group"` + Count uint64 `json:"count"` + Agg int64 `json:"-"` + DecimalAgg float64 `json:"sum"` } var _ GroupCount = GroupCount(groupCountSum{}) var _ GroupCount = GroupCount(groupCountAggregate{}) +var _ GroupCount = GroupCount(groupCountDecimalSum{}) func (g *GroupCount) Clone() (r *GroupCount) { r = &GroupCount{ - Group: make([]FieldRow, len(g.Group)), - Count: g.Count, - Agg: g.Agg, + Group: make([]FieldRow, len(g.Group)), + Count: g.Count, + Agg: g.Agg, + DecimalAgg: g.DecimalAgg, } for i := range g.Group { r.Group[i] = *(g.Group[i].Clone()) @@ -3306,6 +3337,7 @@ func mergeGroupCounts(a, b []GroupCount, limit int) []GroupCount { case 0: a[i].Count += b[j].Count a[i].Agg += b[j].Agg + a[i].DecimalAgg += b[j].DecimalAgg ret = append(ret, a[i]) i++ j++ @@ -4718,25 +4750,25 @@ func (e *executor) executeIntersectShard(ctx context.Context, qcx *Qcx, index st } // executeUnionShard executes a union() call for a local shard. -func (e *executor) executeUnionShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err error) { +func (e *executor) executeUnionShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (out *Row, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard") defer span.Finish() - other := NewRow() + if len(c.Children) == 0 { + return NewRow(), nil + } + if len(c.Children) == 1 { + return e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard) + } + // we have at least two, so... + rows := make([]*Row, len(c.Children)) for i, input := range c.Children { - row, err := e.executeBitmapCallShard(ctx, qcx, index, input, shard) + rows[i], err = e.executeBitmapCallShard(ctx, qcx, index, input, shard) if err != nil { return nil, err } - - if i == 0 { - other = row - } else { - other = other.Union(row) - } } - other.invalidateCount() - return other, nil + return rows[0].Union(rows[1:]...), nil } // executeXorShard executes a xor() call for a local shard. @@ -7939,6 +7971,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool } ret.Count = uint64(result.Count) ret.Agg = result.Val + ret.DecimalAgg = result.FloatVal } } if ret.Count == 0 { diff --git a/executor_internal_test.go b/executor_internal_test.go index 4aa37971c..b752957ef 100644 --- a/executor_internal_test.go +++ b/executor_internal_test.go @@ -28,7 +28,7 @@ import ( func TestExecutor_TranslateRowsOnBool(t *testing.T) { path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-") - holder := NewHolder(path, nil) + holder := NewHolder(path, mustHolderConfig()) defer holder.Close() e := &executor{ diff --git a/executor_test.go b/executor_test.go index 8440adb9d..adcf64dbb 100644 --- a/executor_test.go +++ b/executor_test.go @@ -38,7 +38,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/http" @@ -5151,6 +5151,8 @@ func TestExecutor_GroupByStrings(t *testing.T) { c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "v", pilosa.OptFieldTypeInt(0, 1000)) c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "vv", pilosa.OptFieldTypeInt(0, 1000)) c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "nv", pilosa.OptFieldTypeInt(-1000, 1000)) + c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "dv", pilosa.OptFieldTypeDecimal(2)) + c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "ndv", pilosa.OptFieldTypeDecimal(1)) if err := c.GetNode(0).API.Import(context.Background(), nil, &pilosa.ImportRequest{ Index: "istring", @@ -5167,6 +5169,8 @@ func TestExecutor_GroupByStrings(t *testing.T) { var v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 int64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 var nv1, nv2, nv3, nv4 int64 = -1, -2, -3, -4 + var dv1, dv2, dv3, dv4, dv5, dv6, dv7, dv8, dv9, dv10 int64 = 111, 222, 333, 444, 555, 666, 777, 888, 999, 1000 + var ndv1, ndv2, ndv3, ndv4, ndv5, ndv6, ndv7, ndv8, ndv9, ndv10 int64 = -111, -222, -333, -444, -555, -666, -777, -888, -999, -1000 if err := m0.API.ImportValue(context.Background(), qcx, &pilosa.ImportValueRequest{ Index: "istring", Field: "v", @@ -5197,6 +5201,26 @@ func TestExecutor_GroupByStrings(t *testing.T) { t.Fatalf("importing: %v", err) } + if err := m0.API.ImportValue(context.Background(), qcx, &pilosa.ImportValueRequest{ + Index: "istring", + Field: "dv", + Shard: 0, + ColumnKeys: []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10"}, + Values: []int64{dv1, dv2, dv3, dv4, dv5, dv6, dv7, dv8, dv9, dv10}, + }); err != nil { + t.Fatalf("importing: %v", err) + } + + if err := m0.API.ImportValue(context.Background(), qcx, &pilosa.ImportValueRequest{ + Index: "istring", + Field: "ndv", + Shard: 0, + ColumnKeys: []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10"}, + Values: []int64{ndv1, ndv2, ndv3, ndv4, ndv5, ndv6, ndv7, ndv8, ndv9, ndv10}, + }); err != nil { + t.Fatalf("importing: %v", err) + } + tests := []struct { query string expected []pilosa.GroupCount @@ -5221,6 +5245,20 @@ func TestExecutor_GroupByStrings(t *testing.T) { {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 30}, }, }, + { + query: "GroupBy(Rows(generals), aggregate=Sum(field=dv))", + expected: []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: 2775, DecimalAgg: 27.75}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 3220, DecimalAgg: 32.20}, + }, + }, + { + query: "GroupBy(Rows(generals), aggregate=Sum(field=ndv))", + expected: []pilosa.GroupCount{ + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: -2775, DecimalAgg: -277.5}, + {Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: -3220, DecimalAgg: -322.0}, + }, + }, { query: "GroupBy(Rows(generals), aggregate=Sum(field=v), having=Condition(sum>25))", expected: []pilosa.GroupCount{ @@ -6844,7 +6882,7 @@ func TestMissingKeyRegression(t *testing.T) { c := test.MustRunCluster(t, 1, []server.CommandOption{server.OptCommandServerOptions( pilosa.OptServerStorageConfig(&storage.Config{ Backend: "roaring", - FsyncEnabled: true, + FsyncEnabled: false, }))}) defer c.Close() diff --git a/field.go b/field.go index 793371272..4edccfb3f 100644 --- a/field.go +++ b/field.go @@ -648,7 +648,7 @@ func (f *Field) writeAvailableShards() { func (f *Field) applyTranslateStore() error { // Instantiate & open translation store. var err error - f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1) + f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1, f.holder.cfg.StorageConfig.FsyncEnabled) if err != nil { return errors.Wrap(err, "opening field translate store") } @@ -1098,9 +1098,15 @@ func (f *Field) MutexCheck(ctx context.Context, qcx *Qcx, details bool, limit in if f.Type() != FieldTypeMutex { return nil, errors.New("mutex check only valid for mutex fields") } + + // Rather than deferring the unlock, we grab the standard view + // from the field's viewMap and unlock immediately. This avoids + // holding the rlock for a potentially long time which blocks any + // write lock, and pending write locks block other read locks. f.mu.RLock() - defer f.mu.RUnlock() standard := f.viewMap[viewStandard] + f.mu.RUnlock() + if standard == nil { // no standard view present means we've never needed to create it, // so it has no bits set, so it has no extra bits set. diff --git a/field_internal_test.go b/field_internal_test.go index 88ca4b7d7..080836baa 100644 --- a/field_internal_test.go +++ b/field_internal_test.go @@ -244,6 +244,8 @@ func NewTestField(t testing.TB, opts FieldOption) *TestField { cfg := DefaultHolderConfig() cfg.StorageConfig.Backend = CurrentBackendOrDefault() + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false h := NewHolder(path, cfg) PanicOn(h.Open()) diff --git a/field_test.go b/field_test.go index 38056e8ac..0bc3b60cb 100644 --- a/field_test.go +++ b/field_test.go @@ -159,7 +159,7 @@ func TestField_NameRestriction(t *testing.T) { if err != nil { panic(err) } - field, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", ".meta", pilosa.OptFieldTypeDefault()) + field, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", ".meta", pilosa.OptFieldTypeDefault()) if field != nil { t.Fatalf("unexpected field name %s", err) } @@ -192,13 +192,13 @@ func TestField_NameValidation(t *testing.T) { panic(err) } for _, name := range validFieldNames { - _, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault()) + _, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", name, pilosa.OptFieldTypeDefault()) if err != nil { t.Fatalf("unexpected field name: %s %s", name, err) } } for _, name := range invalidFieldNames { - _, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault()) + _, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", name, pilosa.OptFieldTypeDefault()) if err == nil { t.Fatalf("expected error on field name: %s", name) } diff --git a/fragment.go b/fragment.go index d545a1726..daab8bb34 100644 --- a/fragment.go +++ b/fragment.go @@ -286,10 +286,6 @@ func (f *fragment) Open() error { // Fill cache with rows persisted to disk. f.holder.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard) if err := f.openCache(); err != nil { - e2 := f.closeStorage() - if e2 != nil { - return errors.Wrapf(err, "closing storage: %v, after opening cache", e2) - } return errors.Wrap(err, "opening cache") } diff --git a/fragment_internal_test.go b/fragment_internal_test.go index a7915fcc3..9b4ac461a 100644 --- a/fragment_internal_test.go +++ b/fragment_internal_test.go @@ -3166,7 +3166,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) { origF.Close() fi.Close() - h := NewHolder(fi.Name(), nil) + h := NewHolder(fi.Name(), mustHolderConfig()) PanicOn(h.Open()) idx, err := h.CreateIndex("i", IndexOptions{}) PanicOn(err) @@ -5146,7 +5146,7 @@ func TestImportClearRestart(t *testing.T) { PanicOn(tx2.Commit()) - h3 := NewHolder(filepath.Dir(f2.path()), nil) + h3 := NewHolder(filepath.Dir(f2.path()), mustHolderConfig()) testhook.Cleanup(t, func() { h3.Close() }) diff --git a/go.mod b/go.mod index e590bdd1b..ff7c1d715 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/molecula/featurebase/v2 -replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7 +replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c + +replace go.etcd.io/bbolt => github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 require ( github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d diff --git a/go.sum b/go.sum index 5f4d1be61..0959b010c 100644 --- a/go.sum +++ b/go.sum @@ -230,8 +230,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y= github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s= -github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7 h1:hufElvtCighE0G2VFJYDGWCY8JlmCWZ1FmXvlf25yUQ= -github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI= +github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c h1:YnU+8kIrr/7IDGtIYncawklAs54EWihED4DBIm+kjAA= +github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= @@ -285,6 +285,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 h1:88K0ffxhVphUHxlqW4ewOaXdnJByH4LcCuvYfv0QI/M= +github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA= github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= @@ -344,9 +346,6 @@ github.com/zeebo/blake3 v0.1.1 h1:Nbsts7DdKThRHHd+YNlqiGlRqGEF2bE2eXN+xQ1hsEs= github.com/zeebo/blake3 v0.1.1/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4= github.com/zeebo/pcg v1.0.0 h1:dt+dx+HvX8g7Un32rY9XWoYnd0NmKmrIzpHF7qiTDj0= github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -443,7 +442,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/holder.go b/holder.go index 0ec881add..dea18e3e7 100644 --- a/holder.go +++ b/holder.go @@ -109,7 +109,7 @@ type Holder struct { OpenTransactionStore OpenTransactionStoreFunc // Func to open the ID allocator. - OpenIDAllocator func(string) (*idAllocator, error) + OpenIDAllocator func(string, bool) (*idAllocator, error) // transactionManager transactionManager *TransactionManager @@ -241,7 +241,7 @@ func DefaultHolderConfig() *HolderConfig { OpenTranslateStore: OpenInMemTranslateStore, OpenTranslateReader: nil, OpenTransactionStore: OpenInMemTransactionStore, - OpenIDAllocator: func(string) (*idAllocator, error) { return &idAllocator{}, nil }, + OpenIDAllocator: func(string, bool) (*idAllocator, error) { return &idAllocator{}, nil }, TranslationSyncer: NopTranslationSyncer, Serializer: GobSerializer, Schemator: disco.InMemSchemator, @@ -623,7 +623,7 @@ func (h *Holder) Open() error { h.transactionManager.Log = h.Logger // Open ID allocator. - h.ida, err = h.OpenIDAllocator(filepath.Join(h.path, "idalloc.db")) + h.ida, err = h.OpenIDAllocator(filepath.Join(h.path, "idalloc.db"), h.cfg.StorageConfig.FsyncEnabled) if err != nil { return errors.Wrap(err, "opening ID allocator") } @@ -1472,7 +1472,10 @@ type holderSyncer struct { Cluster *cluster // Translation sync handling. - readers []TranslateEntryReader + readers []TranslateEntryReader + readersMu sync.Mutex + pendingReaders int + stopInitializeReplicationCh chan struct{} syncers errgroup.Group @@ -1596,6 +1599,23 @@ func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) err // resetTranslationSync reinitializes streaming sync of translation data. func (s *holderSyncer) resetTranslationSync() error { + if s.stopInitializeReplicationCh == nil { + // suppose stopTranslationSync[S] holds the lock s.readersMu and tries + // to send a signal to s.stopInitializeRepliationCh. If + // s.stopInitializeReplicationCh is unbufferd and + // s.initializeReplication[I] is trying to acquire s.readersMu, this + // will result in a deadlock. + // Hence, the channel is buffered to prevent this scenario. Once + // [S] releases the lock, [I] will acquire it, however, the value + // of s.pendingReaders will be -1, at this point [I] should not + // attempt to add any more readers since those readers will be 'stale' + // [I] should also drain the channel to prevent the next invocation + // of [I] receiving a stop signal that was meant for the current one + // This is needlessly complicated and is the result of me running + // into various deadlocks while trying to fix handling of column + // key replication. + s.stopInitializeReplicationCh = make(chan struct{}) + } // Stop existing streams. if err := s.stopTranslationSync(); err != nil { return errors.Wrap(err, "stop translation sync") @@ -1655,7 +1675,15 @@ func newActiveTranslationSyncer(ch chan struct{}) *activeTranslationSyncer { // Reset resets the server's translation syncer. func (a *activeTranslationSyncer) Reset() error { - a.ch <- struct{}{} + // just in case some other part of the code has fired + // off a translation sync and it hasn't been received yet + // therefore we don't want to block on send since a.ch is + // (for now) unbuffered. One translationSync is as good as + // another + select { + case a.ch <- struct{}{}: + default: + } return nil } @@ -1665,6 +1693,17 @@ func (a *activeTranslationSyncer) Reset() error { // to complete. This should be called before reconnecting to the cluster in case // of a cluster resize or schema change. func (s *holderSyncer) stopTranslationSync() error { + s.readersMu.Lock() + defer func() { + s.readers = nil // will be populated by initializeReplication + s.readersMu.Unlock() + }() + // send signal to stop initializing more readers + if s.pendingReaders > 0 { + s.pendingReaders = -1 + close(s.stopInitializeReplicationCh) + s.stopInitializeReplicationCh = make(chan struct{}) + } var g errgroup.Group for i := range s.readers { rd := s.readers[i] @@ -1724,7 +1763,6 @@ func (s *holderSyncer) setTranslateReadOnlyFlags(snap *topology.ClusterSnapshot) // replicate these from whichever node is primary for that partition). func (s *holderSyncer) initializeReplication(snap *topology.ClusterSnapshot) error { nodeMaps := make(map[string]TranslateOffsetMap) - if snap.ReplicaN > 1 { if err := s.populateIndexReplication(nodeMaps, snap); err != nil { return err @@ -1734,26 +1772,99 @@ func (s *holderSyncer) initializeReplication(snap *topology.ClusterSnapshot) err return err } + // filter out empty nodes + nodes := make(map[*topology.Node]bool) for _, node := range snap.Nodes { m := nodeMaps[node.ID] - if m.Empty() { - continue + if !m.Empty() { + nodes[node] = true + } + } + + // connect to remote nodes and set up readers + readersCh := make(chan TranslateEntryReader, len(nodes)) + ctx, cancelAddingMoreReaders := context.WithCancel(context.Background()) + defer cancelAddingMoreReaders() + s.readersMu.Lock() + s.pendingReaders = len(nodes) + s.readersMu.Unlock() + go func() { + for { + for node := range nodes { + // check if ctx cancelled + // this means there was a signal sent to stop further init + // of readers + select { + case <-s.Closing: + return + case <-ctx.Done(): + close(readersCh) + return + default: + } + // connect to remote node + m := nodeMaps[node.ID] + rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m) + if err != nil { + continue + } + readersCh <- rd + delete(nodes, node) + } + if len(nodes) == 0 { + close(readersCh) + return + } + time.Sleep(10 * time.Second) + + } + }() + + for { + select { + case <-s.Closing: + cancelAddingMoreReaders() + return nil + case <-s.stopInitializeReplicationCh: + return nil + case rd, ok := <-readersCh: + // all translate readers have been launched, hence channel is + // closed + if !ok { + return nil + } + s.readersMu.Lock() + // [S] has been initiated and acquired the lock first + // at this point we should close the reader we've recieved rather + // than start replication on it. + // [S] should have already closed all the rest + // of the reads if they were still in action. + // we are also draining the channel since the signal for stopping + // further replication was meant for us + if s.pendingReaders == -1 { + rd.Close() + cancelAddingMoreReaders() + drain: + for { + select { + case <-s.stopInitializeReplicationCh: + default: + break drain + } + } + s.readersMu.Unlock() + return nil + } + s.pendingReaders-- + s.readers = append(s.readers, rd) + s.syncers.Go(func() error { + defer rd.Close() + s.readBothTranslateReader(rd, snap) + return nil + }) + s.readersMu.Unlock() } - - // Connect to remote node and begin streaming. - rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m) - if err != nil { - return err - } - s.readers = append(s.readers, rd) - - s.syncers.Go(func() error { - defer rd.Close() - s.readBothTranslateReader(rd, snap) - return nil - }) } - return nil } // populateFieldReplication populates a map from node IDs to TranslateOffsetMaps diff --git a/holder_internal_test.go b/holder_internal_test.go index c87b6bf73..bd35f0b03 100644 --- a/holder_internal_test.go +++ b/holder_internal_test.go @@ -86,6 +86,7 @@ func makeHolder(tb testing.TB, backend string) (*Holder, string, error) { cfg := mustHolderConfig() if backend != "" { cfg.StorageConfig.Backend = backend + cfg.StorageConfig.FsyncEnabled = false } h := NewHolder(path, cfg) return h, path, h.Open() @@ -265,6 +266,8 @@ func mustHolderConfig() *HolderConfig { _ = MustBackendToTxtype(backend) cfg.StorageConfig.Backend = backend } + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false cfg.Schemator = disco.InMemSchemator cfg.Sharder = disco.InMemSharder return cfg diff --git a/holder_test.go b/holder_test.go index 6aeeab79d..4807a997e 100644 --- a/holder_test.go +++ b/holder_test.go @@ -25,11 +25,26 @@ import ( "time" "github.com/molecula/featurebase/v2" + "github.com/molecula/featurebase/v2/disco" "github.com/molecula/featurebase/v2/pql" "github.com/molecula/featurebase/v2/test" "github.com/pkg/errors" ) +// mustHolderConfig provides a default test-friendly holder config. +func mustHolderConfig() *pilosa.HolderConfig { + cfg := pilosa.DefaultHolderConfig() + if backend := pilosa.CurrentBackend(); backend != "" { + _ = pilosa.MustBackendToTxtype(backend) + cfg.StorageConfig.Backend = backend + } + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false + cfg.Schemator = disco.InMemSchemator + cfg.Sharder = disco.InMemSharder + return cfg +} + func TestHolder_Open(t *testing.T) { t.Run("ErrIndexPermission", func(t *testing.T) { if os.Geteuid() == 0 { @@ -283,7 +298,7 @@ func TestHolder_HasData(t *testing.T) { // Note that we are intentionally not using test.NewHolder, // because we want to create a Holder object with an invalid path, // rather than creating a valid holder with a temporary path. - h := pilosa.NewHolder("bad-path", nil) + h := pilosa.NewHolder("bad-path", mustHolderConfig()) if ok, err := h.HasData(); ok || err != nil { t.Fatal("expected HasData to return false, no err, but", ok, err) diff --git a/http/client.go b/http/client.go index fb079218f..1d41a114f 100644 --- a/http/client.go +++ b/http/client.go @@ -32,6 +32,7 @@ import ( pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/encoding/proto" + "github.com/molecula/featurebase/v2/ingest" pnet "github.com/molecula/featurebase/v2/net" "github.com/molecula/featurebase/v2/topology" "github.com/molecula/featurebase/v2/tracing" @@ -284,6 +285,38 @@ func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, in return nil } +// IngestNodeOperations uses the internal/protobuf ingest endpoint for ingest data +func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error { + if uri == nil { + uri = c.defaultURI + } + u := uri.Path(fmt.Sprintf("/internal/ingest/%s/node", indexName)) + + buf, err := c.serializer.Marshal(ireq) + if err != nil { + return errors.Wrap(err, "marshalling") + } + req, err := http.NewRequest("POST", u, bytes.NewReader(buf)) + if err != nil { + return errors.Wrap(err, "creating request") + } + + req.Header.Set("Content-Length", strconv.Itoa(len(buf))) + req.Header.Set("Content-Type", "application/x-protobuf") + req.Header.Set("Accept", "application/x-protobuf") + req.Header.Set("User-Agent", "pilosa/"+pilosa.Version) + + resp, err := c.executeRequest(req.WithContext(ctx)) + if err != nil { + return errors.Wrap(err, "executing request") + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return errors.Errorf("unexpected status code: %s", resp.Status) + } + return nil +} + // MutexCheck uses the mutex-check endpoint to request mutex collision data // from a single node. It produces per-shard results, and does not translate // them. diff --git a/http/handler.go b/http/handler.go index ac5a4dd68..59a3c3162 100644 --- a/http/handler.go +++ b/http/handler.go @@ -42,6 +42,7 @@ import ( "github.com/gorilla/mux" pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/encoding/proto" + "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/pql" "github.com/molecula/featurebase/v2/rbf" @@ -432,7 +433,9 @@ func newRouter(handler *Handler) http.Handler { router.HandleFunc("/internal/index/{index}/shards", handler.handleGetIndexAvailableShards).Methods("GET").Name("GetIndexAvailableShards") router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes") router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client - router.HandleFunc("/internal/ingest/{index}", handler.handleIngestData).Methods("POST").Name("PostIngestData") + router.HandleFunc("/internal/ingest/{index}", handler.handlePostIngestData).Methods("POST").Name("PostIngestData") + router.HandleFunc("/internal/ingest/{index}/node", handler.handlePostIngestNode).Methods("POST").Name("PostIngestNode") + router.HandleFunc("/internal/schema", handler.handleIngestSchema).Methods("POST").Name("PostIngestSchema") router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.handleFindIndexKeys).Methods("POST").Name("FindIndexKeys") router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.handleCreateIndexKeys).Methods("POST").Name("CreateIndexKeys") @@ -524,7 +527,7 @@ func newStatikHandler(h *Handler) statikHandler { func (s statikHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if strings.HasPrefix(r.UserAgent(), "curl") { - msg := "Welcome. Pilosa v" + s.handler.api.Version() + " is running. Visit https://www.pilosa.com/docs/ for more information." + msg := "Welcome. FeatureBase v" + s.handler.api.Version() + " is running. Visit https://docs.molecula.cloud for more information." if s.statikFS != nil { msg += " Try the Web UI by visiting this URL in your browser." } @@ -1337,7 +1340,9 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) { resp.write(w, err) } -func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) { +// handlePostIngestData handles JSON ingest data that may need key +// translation, for the entire cluster. +func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) { if !validHeaderAcceptJSON(r.Header) { http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable) return @@ -1351,6 +1356,15 @@ func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) { qcx := h.api.Txf().NewQcx() err := h.api.IngestOperations(r.Context(), qcx, indexName, r.Body) + if err == nil { + err = qcx.Finish() + if err != nil { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + return + } + } else { + qcx.Abort() + } resp := successResponse{h: h, Name: indexName} resp.write(w, err) @@ -2908,6 +2922,52 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request } } +// handlePostIngestNode is the internal endpoint taking already-translated +// ingest operations, sorted by shard, for a single node. +func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) { + // Verify that request is only communicating over protobufs. + if error, code := validateProtobufHeader(r); error != "" { + http.Error(w, error, code) + return + } + + ctx := r.Context() + + // Read entire body. + span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body") + body, err := readBody(r) + span.LogKV("bodySize", len(body)) + span.Finish() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + req := &ingest.ShardedRequest{} + span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal") + err = proto.DefaultSerializer.Unmarshal(body, req) + span.Finish() + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + urlVars := mux.Vars(r) + indexName := urlVars["index"] + + qcx := h.api.Txf().NewQcx() + err = h.api.IngestNodeOperations(r.Context(), qcx, indexName, req) + if err == nil { + err = qcx.Finish() + if err != nil { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + } + } else { + http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError) + qcx.Abort() + } +} + func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) { // Verify that request is only communicating over protobufs. if r.Header.Get("Content-Type") != "application/x-protobuf" { diff --git a/idalloc.go b/idalloc.go index 4c6ea423f..ffa07baf8 100644 --- a/idalloc.go +++ b/idalloc.go @@ -53,18 +53,20 @@ func (k IDAllocKey) String() string { } type idAllocator struct { - db *bolt.DB + db *bolt.DB + fsyncEnabled bool } -type OpenIDAllocatorFunc func(path string) (*idAllocator, error) // whyyyyyyyyy +type OpenIDAllocatorFunc func(path string, enableFsync bool) (*idAllocator, error) // whyyyyyyyyy -func OpenIDAllocator(path string) (*idAllocator, error) { - db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second}) +func OpenIDAllocator(path string, enableFsync bool) (*idAllocator, error) { + db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !enableFsync}) if err != nil { return nil, err } - return &idAllocator{db}, nil + return &idAllocator{db: db, fsyncEnabled: enableFsync}, nil } + func (ida *idAllocator) Replace(reader io.Reader) error { newFile := ida.db.Path() + ".bak" liveFile := ida.db.Path() @@ -92,7 +94,7 @@ func (ida *idAllocator) Replace(reader io.Reader) error { } else { _ = os.Remove(liveFile + ".sav") } - db, err := bolt.Open(liveFile, 0666, &bolt.Options{Timeout: 1 * time.Second}) + db, err := bolt.Open(liveFile, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !ida.fsyncEnabled}) ida.db = db return err } diff --git a/idalloc_test.go b/idalloc_test.go index dff559146..b28a56604 100644 --- a/idalloc_test.go +++ b/idalloc_test.go @@ -40,7 +40,7 @@ func TestIDAlloc(t *testing.T) { }() // Open bolt. - db, err := bolt.Open(f.Name(), 0666, &bolt.Options{Timeout: 1 * time.Second}) + db, err := bolt.Open(f.Name(), 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: true}) if err != nil { t.Errorf("opening bolt: %v", err) return diff --git a/index.go b/index.go index 25b7bd5ee..83a4e8e4a 100644 --- a/index.go +++ b/index.go @@ -249,7 +249,7 @@ func (i *Index) open(idx *disco.Index) (err error) { partitionID := partitionID g.Go(func() error { - store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN) + store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN, i.holder.cfg.StorageConfig.FsyncEnabled) if err != nil { return errors.Wrapf(err, "opening index translate store: partition=%d", partitionID) } diff --git a/index_internal_test.go b/index_internal_test.go index a8cd51706..0bf4f6909 100644 --- a/index_internal_test.go +++ b/index_internal_test.go @@ -26,7 +26,7 @@ func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index { if err != nil { panic(err) } - h := NewHolder(path, nil) + h := NewHolder(path, mustHolderConfig()) index, err := h.CreateIndex("i", opt) testhook.Cleanup(tb, func() { h.Close() diff --git a/index_test.go b/index_test.go index 149133108..890d481d1 100644 --- a/index_test.go +++ b/index_test.go @@ -261,7 +261,7 @@ func TestIndex_InvalidName(t *testing.T) { if err != nil { panic(err) } - index, err := pilosa.NewIndex(pilosa.NewHolder(path, nil), path, "ABC") + index, err := pilosa.NewIndex(pilosa.NewHolder(path, mustHolderConfig()), path, "ABC") if err == nil { t.Fatalf("should have gotten an error on index name with caps") } diff --git a/ingest/codec.go b/ingest/codec.go index c966e3f1c..91b63000f 100644 --- a/ingest/codec.go +++ b/ingest/codec.go @@ -15,10 +15,13 @@ package ingest import ( + "bytes" + "encoding/json" "fmt" "io" "io/ioutil" "math" + "sort" "strconv" "time" @@ -37,15 +40,22 @@ import ( // decimal yes yes no // timestamp yes yes no -type KeyLookupFunc func(...string) (map[string]uint64, error) +// KeyTranslator is a thing that can translate strings to IDs, and also +// IDs back to strings. The ID->string conversion is used only to render +// a request back to JSON, which in turn is only used in testing. The +// functions optionally take an existing map which they then augment. +type KeyTranslator interface { + TranslateKeys(keys ...string) (map[string]uint64, error) + TranslateIDs(ids ...uint64) (map[uint64]string, error) +} // Codec is a single-use parser which decodes data into columnar vectors. type Codec interface { - AddSetField(name string, lookup KeyLookupFunc) error - AddTimeQuantumField(name string, lookup KeyLookupFunc) error - AddMutexField(name string, lookup KeyLookupFunc) error + AddSetField(name string, keys KeyTranslator) error + AddTimeQuantumField(name string, keys KeyTranslator) error + AddMutexField(name string, keys KeyTranslator) error AddBoolField(name string) error - AddIntField(name string, lookup KeyLookupFunc) error + AddIntField(name string, keys KeyTranslator) error AddDecimalField(name string, scale int64) error AddTimestampField(name string, scale time.Duration, epoch int64) error @@ -56,138 +66,299 @@ type Codec interface { type jsonDecFn func(recID uint64, typ jsonparser.ValueType, data []byte) error -// applyTranslationFn is a function which applies key lookups to the values -// of an operation, meaning it needs to know whether it's applying them -// to the signed or unsigned values. -type applyTranslationFn func(*FieldOperation, []uint64) error +// jsonEncFn encodes a value, or range of values, according to a given +// fieldCodec's translation rules. key values, if present, will be used to +// replace whichever values could have been provided as keys. so for instance, +// with an int field which is keyed, values are actually of type int64, but +// if strings are present, those are used. for a time quantum field, the time +// is set from the signed field, and the value is replaced by the key if +// keys are provided. +type jsonEncFn func(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error -type jsonFieldCodec struct { +// fieldCodec represents something that can encode and decode a +// particular field. Its methods are not reentrant, it uses internal +// buffers. +type fieldCodec struct { valueKeys *StringTable currentOp *FieldOperation decode jsonDecFn - translate applyTranslationFn + encode jsonEncFn // For timestamp: scale-in-nanoseconds; for instance, if scaleUnit is // 1,000,000,000, we are storing numbers-of-seconds since the Unix epoch. // The actual value recorded in BSI will be offset by the field's // epoch, but we don't need to know that. // For decimal: Decimal digits of precision. So for instance, with - // scaleUnit 2, "1" is stored as 100 and "1.2" is stored as 120. + // scale 2, scaleUnit is 100, "1" is stored as 100 and "1.2" is stored as + // 120. scaleUnit int64 + scale int64 epoch int64 // used only by Timestamp fields scratch []uint64 // reusable scratch space for sets of values - lookup KeyLookupFunc + keys KeyTranslator + buf []byte // scratch space for format operations. fieldType FieldType } // JSONCodec is a Codec which accepts a JSON map of record keys/ids to updated value maps. type JSONCodec struct { - recKeys *StringTable - fields map[string]*jsonFieldCodec - keyLookup KeyLookupFunc - currentOp *Operation + fieldTypes map[string]FieldType + recKeys *StringTable + fields map[string]*fieldCodec + keys KeyTranslator + currentOp *Operation +} + +// jsonBuffer is a bytes.Buffer which has an associated json.Encoder which +// can be used to write to its buffer. Both types are just embedded because +// they have non-overlapping APIs. Please don't look at my horrible face. +// +// This has some internal objects it can use to stash things +// so that it can pass &foo to enc.Encode() without needing to heap-allocate +// something for the interface conversion, and a buffer it can use for +// converting numbers or times. +// +// It also has an internal error buffer. In fact, in many cases, errors +// are so far as I can tell absolutely impossible -- bytes.Buffer specifically +// promises never to yield an error, and nothing seems to hint that +// enc.Encode can error on integers, strings, booleans, or arrays. So we +// have dozens of error checks that we can't cause to happen even with +// malformed inputs... so we eat those errors, don't require them to be +// checked externally, and return them when done if anyone checks. +type jsonBuffer struct { + *bytes.Buffer + enc *json.Encoder + buf [48]byte // scratch space for using strconv.AppendInt, etc + uintbuf []uint64 // dummy buffer so that we don't have to alloc to print []uint64 + strbuf []string // dummy buffer so that we don't have to alloc to print []string + str string // and again, "so we don't have to alloc a copy" + err error +} + +// Encode removes the stray newlines added by json.Encoder. +func (j *jsonBuffer) Encode(v interface{}) { + err := j.enc.Encode(v) + if err == nil { + j.Truncate(j.Len() - 1) + } else { + j.err = err + } +} + +// EncodeInt uses AppendInt into a static buffer to reduce allocs. +func (j *jsonBuffer) EncodeInt(i int64) { + rep := strconv.AppendInt(j.buf[:0], i, 10) + _, _ = j.Write(rep) +} + +// EncodeUint uses AppendUint into a static buffer to reduce allocs. +func (j *jsonBuffer) EncodeUint(u uint64) { + rep := strconv.AppendUint(j.buf[:0], u, 10) + _, _ = j.Write(rep) +} + +// EncodeUints uses a static copy of a []uint64 -- the slice, not its +// contents -- in already-allocated memory so the interface conversion +// doesn't have to do that. +func (j *jsonBuffer) EncodeUints(u []uint64) { + j.uintbuf = u + j.Encode(&j.uintbuf) + j.strbuf = nil +} + +// EncodeStrings uses a static copy of a []string -- the slice, not its +// contents -- in already-allocated memory so the interface conversion +// doesn't have to do that. +func (j *jsonBuffer) EncodeStrings(s []string) { + j.strbuf = s + j.Encode(&j.strbuf) + j.strbuf = nil +} + +// EncodeQuotedUint uses AppendUint into a static buffer to reduce allocs, +// while also surrounding the value with quotes. +func (j *jsonBuffer) EncodeQuotedUint(u uint64) { + j.buf[0] = '"' + rep := strconv.AppendUint(j.buf[1:1], u, 10) + j.buf[len(rep)+1] = '"' + _, _ = j.Write(j.buf[:len(rep)+2]) +} + +// EncodeString appends the JSON encoding of a string. This exists +// because otherwise runtime allocates a heap-allocated copy of the +// string to live inside an interface{} for the duration of a function +// call... +func (j *jsonBuffer) EncodeString(s string) { + j.str = s + j.Encode(&j.str) + j.str = "" +} + +// EncodeTime exists because time's MarshalJSON allocates a new +// buffer every time it gets called, resulting in a full 13% of +// all the allocations produced in a test run, plus another 13% +// or so of them which were for the copies of the time objects +// made to stuff them into an interface{}. Eww. +func (j *jsonBuffer) EncodeTime(t time.Time) { + j.buf[0] = '"' + rep := t.AppendFormat(j.buf[1:1], time.RFC3339Nano) + j.buf[len(rep)+1] = '"' + _, _ = j.Write(j.buf[:len(rep)+2]) +} + +// EncodeBool writes a literal representation directly to reduce allocs. +func (j *jsonBuffer) EncodeBool(b bool) { + if b { + _, _ = j.WriteString("true") + } else { + _, _ = j.WriteString("false") + } +} + +func (j *jsonBuffer) Err() error { + return j.err +} + +func newJSONBuffer(data []byte) *jsonBuffer { + e := &jsonBuffer{Buffer: bytes.NewBuffer(data)} + e.enc = json.NewEncoder(e.Buffer) + e.enc.SetEscapeHTML(false) // we are not doing HTML, just JSON + return e } var _ Codec = &JSONCodec{} -func NewJSONCodec(lookup KeyLookupFunc) (*JSONCodec, error) { - j := &JSONCodec{fields: map[string]*jsonFieldCodec{}} - if lookup != nil { +func NewJSONCodec(keys KeyTranslator) (*JSONCodec, error) { + j := &JSONCodec{ + fields: map[string]*fieldCodec{}, + fieldTypes: map[string]FieldType{}, + } + if keys != nil { j.recKeys = NewStringTable() - j.keyLookup = lookup + j.keys = keys } return j, nil } -func (codec *JSONCodec) AddTimeQuantumField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} +func (codec *JSONCodec) AddTimeQuantumField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeTimeQuantum, + keys: keys, + } fieldCodec.decode = fieldCodec.DecodeTimeQuantumValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateUnsigned - } - return codec.addField(name, FieldTypeTimeQuantum, fieldCodec, lookup) + fieldCodec.encode = fieldCodec.EncodeTimeQuantumValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) AddSetField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} +func (codec *JSONCodec) AddSetField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeSet, + keys: keys, + } fieldCodec.decode = fieldCodec.DecodeSetValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateUnsigned - } - return codec.addField(name, FieldTypeSet, fieldCodec, lookup) + fieldCodec.encode = fieldCodec.EncodeSetValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) AddIntField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} +func (codec *JSONCodec) AddIntField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeInt, + keys: keys, + } fieldCodec.decode = fieldCodec.DecodeIntValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateSigned - } - return codec.addField(name, FieldTypeInt, fieldCodec, lookup) + fieldCodec.encode = fieldCodec.EncodeIntValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) AddMutexField(name string, lookup KeyLookupFunc) error { - fieldCodec := &jsonFieldCodec{} - fieldCodec.decode = fieldCodec.DecodeMutexValue - if lookup != nil { - fieldCodec.translate = (*FieldOperation).TranslateUnsigned +func (codec *JSONCodec) AddMutexField(name string, keys KeyTranslator) error { + fieldCodec := &fieldCodec{ + fieldType: FieldTypeMutex, + keys: keys, } - return codec.addField(name, FieldTypeMutex, fieldCodec, lookup) + fieldCodec.decode = fieldCodec.DecodeMutexValue + fieldCodec.encode = fieldCodec.EncodeMutexValue + return codec.addField(name, fieldCodec) } func (codec *JSONCodec) AddBoolField(name string) error { - fieldCodec := &jsonFieldCodec{} + fieldCodec := &fieldCodec{ + fieldType: FieldTypeBool, + } fieldCodec.decode = fieldCodec.DecodeBoolValue - return codec.addField(name, FieldTypeBool, fieldCodec, nil) + fieldCodec.encode = fieldCodec.EncodeBoolValue + return codec.addField(name, fieldCodec) } // TimestampField is used to store seconds since unix epoch. The numeric values // stored are adjusted based on the given time.Duration; for instance, if the // time scale is time.Second, then the second after the epoch is stored as 1, -// if it's time.Millisecond, then it's stored as 1000, etcetera. +// if it's time.Millisecond, then it's stored as 1000, etcetera. The epoch +// passed to this function should be the offset from the Unix epoch to the +// desired epoch, in the same scale. (So if the scale is milliseconds, +// it should be the Unix timestamp in seconds, times 1000.) func (codec *JSONCodec) AddTimestampField(name string, timeScale time.Duration, epoch int64) error { - fieldCodec := &jsonFieldCodec{scaleUnit: int64(timeScale), epoch: epoch} + fieldCodec := &fieldCodec{ + fieldType: FieldTypeTimeStamp, + scaleUnit: int64(timeScale), + epoch: epoch, + } fieldCodec.decode = fieldCodec.DecodeTimeValue - return codec.addField(name, FieldTypeTimeStamp, fieldCodec, nil) + fieldCodec.encode = fieldCodec.EncodeTimeValue + return codec.addField(name, fieldCodec) } // AddDecimalField adds a decimal field, which is stored as integer values // with a scale offset, but parsed as floating point values. For instance, // with decimalScale=2, `0.01` would store the value 1. func (codec *JSONCodec) AddDecimalField(name string, decimalScale int64) error { - fieldCodec := &jsonFieldCodec{scaleUnit: int64(math.Pow10(int(decimalScale)))} + fieldCodec := &fieldCodec{ + fieldType: FieldTypeDecimal, + scale: decimalScale, + scaleUnit: int64(math.Pow(10, float64(decimalScale))), + } fieldCodec.decode = fieldCodec.DecodeDecimalValue - return codec.addField(name, FieldTypeDecimal, fieldCodec, nil) + fieldCodec.encode = fieldCodec.EncodeDecimalValue + return codec.addField(name, fieldCodec) } -func (codec *JSONCodec) addField(name string, fieldType FieldType, fieldCodec *jsonFieldCodec, lookup KeyLookupFunc) error { +func (codec *JSONCodec) addField(name string, fieldCodec *fieldCodec) error { if _, ok := codec.fields[name]; ok { return fmt.Errorf("duplicate field %q", name) } - if lookup != nil { + if fieldCodec.keys != nil { fieldCodec.valueKeys = NewStringTable() - fieldCodec.lookup = lookup } - fieldCodec.fieldType = fieldType codec.fields[name] = fieldCodec + codec.fieldTypes[name] = fieldCodec.fieldType return nil } // decodeSetOrValue decodes a value which might be either an array of values or a // single value, where values might be string keys or bare numbers, calling cb for // each value it finds. -func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) { +func (j *fieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) { switch dataType { case jsonparser.Array: _, arrayErr := jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) { - id, idErr := j.valueKeys.ID(value) - if idErr != nil { - err = idErr - return - } - // stash an error if we got one - valueErr := cb(id) - if valueErr != nil { - err = valueErr + var id uint64 + switch dataType { + case jsonparser.String: + id, err = j.valueKeys.ID(value) + if err != nil { + return + } + err = cb(id) + case jsonparser.Number: + if j.valueKeys != nil { + err = errors.New("expecting key, got numeric value") + return + } + id, err = strconv.ParseUint(pretendByteIsString(value), 10, 64) + if err != nil { + return + } + err = cb(id) + default: + err = fmt.Errorf("expecting value or array, got %v", dataType) } }) if err != nil { @@ -212,23 +383,32 @@ func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data [] } return cb(id) default: - return fmt.Errorf("expecting array, got %v", dataType) + return fmt.Errorf("expecting value or array, got %v", dataType) } return err } // DecodeSetValue decodes a set of unsigned values from the provided data into the // associated currentOp. -func (j *jsonFieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { +func (j *fieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { return j.decodeSetOrValue(dataType, data, func(id uint64) error { j.currentOp.AddPair(recID, id) return nil }) } +func (j *fieldCodec) EncodeSetValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(keys) == 1 || len(values) == 1 { + // simplify: just hand this off to a single-value case + return j.EncodeMutexValue(dst, values, signed, keys) + } + appendKeysJSON(dst, values, keys) + return nil +} + // DecodeIntValue decodes a single signed value from the provided data // into the associated currentOp. -func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { switch dataType { case jsonparser.String: value, err := j.valueKeys.IntID(data) @@ -255,25 +435,62 @@ func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueT return nil } +func (j *fieldCodec) EncodeIntValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(keys) == 0 { + if len(signed) == 0 { + return errors.New("encodeIntValue: need a value") + } + dst.EncodeInt(signed[0]) + return nil + } + dst.EncodeString(keys[0]) + return nil +} + // DecodeMutexValue decodes a single unsigned value from the provided data // into the associated currentOp. -func (j *jsonFieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { switch dataType { - case jsonparser.Number, jsonparser.String: - id, err := j.valueKeys.ID(data) + case jsonparser.String: + value, err := j.valueKeys.ID(data) if err != nil { return err } - j.currentOp.AddPair(recID, id) + j.currentOp.AddPair(recID, value) + case jsonparser.Number: + if j.valueKeys != nil { + return errors.New("expecting string key, got numeric value") + } + value, err := strconv.ParseUint(pretendByteIsString(data), 10, 64) + if err != nil { + return err + } + j.currentOp.AddPair(recID, value) default: - return fmt.Errorf("expecting integer value, got %v", dataType) + if j.valueKeys != nil { + return fmt.Errorf("expecting string key, got %v", dataType) + } else { + return fmt.Errorf("expecting integer value, got %v", dataType) + } } return nil } +func (j *fieldCodec) EncodeMutexValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(keys) == 0 { + if len(values) == 0 { + return errors.New("encodeMutexValue: need a value") + } + dst.EncodeUint(values[0]) + return nil + } + dst.EncodeString(keys[0]) + return nil +} + // DecodeBoolValue decodes a single true/false value from the provided data // into the associated currentOp. -func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error { value := uint64(0) switch typ { case jsonparser.String: @@ -303,9 +520,21 @@ func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, return nil } +func (j *fieldCodec) EncodeBoolValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(values) == 0 { + return errors.New("encoding boolean value, but none provided") + } + var x bool + if values[0] != 0 { + x = true + } + dst.EncodeBool(x) + return nil +} + // DecodeTimeQuantumValue decodes a timestamp, and a set of bits from the // provided data into the associated currentOp. -func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error { j.scratch = j.scratch[:0] stamp := time.Unix(0, 0).UTC() err := jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) { @@ -346,8 +575,27 @@ func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.Val return nil } +// Our format does not allow a record to have multiple values set with +// different timestamps at the same time. We use the first timestamp for +// the whole set. +func (j *fieldCodec) EncodeTimeQuantumValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(values) == 0 { + return errors.New("encoding time quantum value: no value provided") + } + dst.WriteString(`{"values":`) + appendKeysJSON(dst, values, keys) + // a zero timestamp is idiomatic for no-time-provided, and i sort of + // hate that, but here we are. + if len(signed) > 0 && signed[0] != 0 { + _, _ = dst.WriteString(`,"time":`) + dst.EncodeTime(time.Unix(0, signed[0]).UTC()) + } + _, _ = dst.WriteString(`}`) + return nil +} + // DecodeTimeValue will eventually work but right now it doesn't actually. -func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { +func (j *fieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) { var stamp time.Time switch dataType { case jsonparser.String: @@ -364,12 +612,22 @@ func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.Value return fmt.Errorf("parsing numeric timestamp: %w", err) } j.currentOp.AddSignedPair(recID, i64) + default: + return fmt.Errorf("expecting time, got %s", dataType) } return nil } +func (j *fieldCodec) EncodeTimeValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + if len(signed) == 0 { + return errors.New("encoding time value: no value provided") + } + dst.EncodeTime(time.Unix(0, (signed[0]+j.epoch)*j.scaleUnit).UTC()) + return nil +} + // DecodeDecimalValue will eventually work but right now it doesn't actually. -func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { +func (j *fieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error { switch dataType { case jsonparser.String, jsonparser.Number: value, err := jsonparser.GetFloat(data) @@ -384,6 +642,28 @@ func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.Va return nil } +func (j *fieldCodec) EncodeDecimalValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error { + j.buf = strconv.AppendInt(j.buf[:0], signed[0], 10) + scale := int(j.scale) + if len(j.buf) > scale { + j.buf = append(j.buf, '.') + // shove the last scaleUnit values over + copy(j.buf[len(j.buf)-scale:], j.buf[len(j.buf)-scale-1:]) + j.buf[len(j.buf)-scale-1] = '.' + } + _, _ = dst.Write(j.buf) + return nil +} + +// FieldTypes gives a mapping of fields to their basic types used by this +// codec. +func (codec *JSONCodec) FieldTypes() map[string]FieldType { + return codec.fieldTypes +} + +// ParseKeyedRecords parses the records it finds. Note that, when you're doing +// a Write op, this will update ClearRecordIDs automatically as it goes, +// even though record_ids isn't specified in the JSON for that case. func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) { seen := make(map[uint64]struct{}) return jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { @@ -412,8 +692,8 @@ func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) { }) } -func (codec *JSONCodec) ParseOperation(data []byte) (op *Operation, err error) { - op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields))} +func (codec *JSONCodec) ParseOperation(data []byte, seq int) (op *Operation, err error) { + op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields)), Seq: seq} codec.currentOp = op err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error { switch string(key) { @@ -491,10 +771,12 @@ func (codec *JSONCodec) Parse(r io.Reader) (req *Request, err error) { func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { var ops []*Operation var lastErr error + var seq int _, err = jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) { switch dataType { case jsonparser.Object: - op, err := codec.ParseOperation(value) + op, err := codec.ParseOperation(value, seq) + seq++ if err != nil { lastErr = fmt.Errorf("parsing operation: %v", err) return @@ -512,34 +794,39 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { } // and now, key translation! var keyMap []uint64 - if codec.keyLookup != nil { - keyMap, err = MapForStringTable(codec.recKeys, codec.keyLookup) + if codec.keys != nil { + keyMap, err = codec.recKeys.MakeIDMap(codec.keys) if err != nil { return nil, fmt.Errorf("trying to find record key mapping: %w", err) } } - req = &Request{FieldTypes: make(map[string]FieldType, len(codec.fields))} + req = &Request{} valueMaps := map[string]func(*FieldOperation) error{} for name, fieldCodec := range codec.fields { // make closure survive iteration fieldCodec := fieldCodec - if fieldCodec.lookup != nil { - fieldMap, err := MapForStringTable(fieldCodec.valueKeys, fieldCodec.lookup) + if fieldCodec.keys != nil { + fieldMap, err := fieldCodec.valueKeys.MakeIDMap(fieldCodec.keys) if err != nil { return nil, fmt.Errorf("trying to find value mapping for %q: %w", name, err) } - valueMaps[name] = func(fo *FieldOperation) error { - return fieldCodec.translate(fo, fieldMap) + if fieldCodec.fieldType == FieldTypeInt { + valueMaps[name] = func(fo *FieldOperation) error { + return translateSigned(fieldMap, fo.Signed) + } + } else { + valueMaps[name] = func(fo *FieldOperation) error { + return translateUnsigned(fieldMap, fo.Values) + } } } - req.FieldTypes[name] = fieldCodec.fieldType } for _, op := range ops { // For Clear and Write, we need to translate/sort our record ID // list. if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { if keyMap != nil { - if err = translateUnsignedSlice(op.ClearRecordIDs, keyMap); err != nil { + if err = translateUnsigned(keyMap, op.ClearRecordIDs); err != nil { return nil, fmt.Errorf("mapping record keys for clear op: %w", err) } } @@ -554,7 +841,7 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { continue } if keyMap != nil { - if err = fieldOp.TranslateKeys(keyMap); err != nil { + if err = translateUnsigned(keyMap, fieldOp.RecordIDs); err != nil { return nil, fmt.Errorf("mapping record keys for op on %q: %w", field, err) } } @@ -574,6 +861,283 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) { return req, nil } +// AppendBytes appends bytes which we would expect to produce the same +// request, using this codec. It does not try to recreate FieldTypes, which +// would be handled by the codec anyway. It's written as an Append so you +// can reuse a buffer. +func (codec *JSONCodec) AppendBytes(req *Request, data []byte) (out []byte, err error) { + if req == nil || len(req.Ops) == 0 { + return append(data, "[]"...), nil + } + dst := newJSONBuffer(data) + // we ignore the FieldTypes part of the Request, which is just there to + // let the request's ByShard use the correct sorting routines, which is + // itself sort of awful. The codec will insert it again when parsing the + // bytes. + _, _ = dst.WriteString(`,`) + for _, op := range req.Ops { + // EncodeJSON needs to have access to this codec's field data + // and key translators. + err = op.EncodeJSON(dst, codec) + if err != nil { + return nil, err + } + _, _ = dst.WriteString(`,`) + } + // delete the last comma + dst.Truncate(dst.Len() - 1) + _, _ = dst.WriteString(`]`) + // there's very few ways an error could occur, possibly none, but + // just in case lots of internal operations stashed an error if one + // happened, so we'll return anything they came up with. + return dst.Bytes(), dst.Err() +} + +// appendIDsJSON appends the provided IDs to a JSON stream as a bracketed +// list of quoted strings if there's a key translator, or integer values +// if the KeyTranslator is nil, comma-separated. It can error because a +// translator can error. +func (codec *JSONCodec) appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { + return appendIDsJSON(dst, values, keys) +} + +func appendKeysJSON(dst *jsonBuffer, values []uint64, keys []string) { + if len(values) == 0 && len(keys) == 0 { + _, _ = dst.WriteString("[]") + return + } + if len(keys) != 0 { + dst.EncodeStrings(keys) + } else { + dst.EncodeUints(values) + } +} + +func appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error { + if len(values) == 0 { + _, _ = dst.WriteString("[]") + return nil + } + + if keys == nil { + dst.EncodeUints(values) + return nil + } + _, _ = dst.WriteString(`[`) + translated, err := keys.TranslateIDs(values...) + if err != nil { + return err + } + for _, v := range values { + dst.EncodeString(translated[v]) + _, _ = dst.WriteString(`,`) + } + dst.Truncate(dst.Len() - 1) + _, _ = dst.WriteString(`]`) + return nil +} + +// RequestByShard makes up for the fact that we don't want to stash the +// field type data in the request, but we need it to actually do the by-shard. +func (codec *JSONCodec) RequestByShard(req *Request) (*ShardedRequest, error) { + return req.ByShard(codec.fieldTypes) +} + +// EncodeJSON encodes an operation as JSON using a provided buffer. +// It uses the provided codec where necessary to help with key +// translation. +func (o *Operation) EncodeJSON(dst *jsonBuffer, codec *JSONCodec) (err error) { + // We're not trying to guarantee that what we produce makes sense or is + // identical to what produced us, just to write our current state out. + if o == nil { + _, _ = dst.WriteString(`{}`) + return nil + } + _, _ = dst.WriteString(`{"action":`) + dst.EncodeString(o.OpType.String()) + // it is intentional that o.Seq isn't encoded here; it makes no sense + // to allow an op to specify its seq in JSON. + if o.OpType != OpWrite { + // for Write, ClearRecordIDs and ClearFields were computed + // from the records being written, and aren't actually part of the data. + if len(o.ClearRecordIDs) != 0 { + _, _ = dst.WriteString(`,"record_ids":`) + err = codec.appendIDsJSON(dst, o.ClearRecordIDs, codec.keys) + if err != nil { + return err + } + } + if len(o.ClearFields) != 0 { + _, _ = dst.WriteString(`,"fields":`) + dst.EncodeStrings(o.ClearFields) + } + } + if len(o.FieldOps) == 0 { + _, _ = dst.WriteString(`}`) + return + } + // collect all the fields that have a non-empty set of records. we can + // just ignore the others. + fieldNames := make([]string, 0, len(o.FieldOps)) + for field, op := range o.FieldOps { + if op != nil && len(op.RecordIDs) != 0 { + fieldNames = append(fieldNames, field) + } + } + // nevermind then + if len(fieldNames) == 0 { + _, _ = dst.WriteString(`}`) + return nil + } + _, _ = dst.WriteString(`,"records":{`) + + // now we have to invert the logic, creating records from + // fields with corresponding ops. uh-oh. + fieldOps := make([]*FieldOperation, len(o.FieldOps)) + fieldCodecs := make([]*fieldCodec, len(o.FieldOps)) + fieldKeys := make([][]string, len(o.FieldOps)) // translated field keys + indexes := make([]int, len(o.FieldOps)) + sort.Slice(fieldNames, func(i, j int) bool { return fieldNames[i] < fieldNames[j] }) + next := ^uint64(0) + var idKeys map[uint64]string + if codec.keys != nil { + idKeys = make(map[uint64]string) + } + for i, field := range fieldNames { + // populate a parallel slice of field ops so we don't have + // to do map lookups for every single piece of data + op := o.FieldOps[field] + fieldOps[i] = op + fc := codec.fields[field] + if fc == nil { + return fmt.Errorf("unknown field: %q", field) + } + fieldCodecs[i] = fc + if codec.keys != nil { + // we'll build a list of keys we need for any records + for _, v := range op.RecordIDs { + idKeys[v] = "" + } + } + id := op.RecordIDs[0] + if id < next { + next = id + } + if fc.keys != nil { + thisFieldKeys := make([]string, len(op.RecordIDs)) + if fc.fieldType == FieldTypeInt { + u := make([]uint64, len(op.Signed)) + for i := range op.Signed { + u[i] = uint64(op.Signed[i]) + } + valueKeys, err := fc.keys.TranslateIDs(u...) + if err != nil { + return err + } + for j, v := range op.Signed { + thisFieldKeys[j] = valueKeys[uint64(v)] + } + } else { + valueKeys, err := fc.keys.TranslateIDs(op.Values...) + if err != nil { + return err + } + for j, v := range op.Values { + thisFieldKeys[j] = valueKeys[v] + } + } + fieldKeys[i] = thisFieldKeys + } + } + // ... no fieldops actually have any entries. therefore no record will + // have any fields set, therefore no records exist... + if next == ^uint64(0) { + _, _ = dst.WriteString(`}}`) + return nil + } + if codec.keys != nil { + idList := make([]uint64, 0, len(idKeys)) + for k := range idKeys { + idList = append(idList, k) + } + idKeys, err = codec.keys.TranslateIDs(idList...) + if err != nil { + return err + } + } + + // next is the ID of a record which exists for at least one field. + // we'll recompute it every loop. + for next != ^uint64(0) { + if codec.keys != nil { + dst.EncodeString(idKeys[next]) + } else { + dst.EncodeQuotedUint(next) + } + _, _ = dst.WriteString(`:{`) + current := next + next = ^uint64(0) + for i, field := range fieldNames { + op := fieldOps[i] + idx := indexes[i] + if idx >= len(op.RecordIDs) { + continue + } + id := op.RecordIDs[idx] + if id == current { + var j int + // count ahead to first index which is either outside the list + // or a different id + for j = idx; j < len(op.RecordIDs) && op.RecordIDs[j] == id; j++ { + } + // fmt.Printf("field %s encoding %d-%d (v %d, s %d, k %d)\n", + // field, idx, j, len(op.Values), len(op.Signed), len(fieldKeys[i])) + // print this one, and advance this index to next position + dst.EncodeString(field) + _, _ = dst.WriteString(`:`) + var values []uint64 + var signed []int64 + var keys []string + if len(op.Values) >= j { + values = op.Values[idx:j] + } + if len(op.Signed) >= j { + signed = op.Signed[idx:j] + } + if len(fieldKeys[i]) >= j { + keys = fieldKeys[i][idx:j] + } + err = fieldCodecs[i].encode(dst, values, signed, keys) + if err != nil { + return err + } + _, _ = dst.WriteString(`,`) + indexes[i] = j + // and we'll fall through to the id < next check, so we + // just set id here. + if indexes[i] < len(op.RecordIDs) { + id = op.RecordIDs[indexes[i]] + } else { + id = ^uint64(0) + } + } + if id < next { + next = id + } + } + // we should always have a trailing comma after any entry, and + // if there were no entries we shouldn't have had a list... + dst.Truncate(dst.Len() - 1) + _, _ = dst.WriteString(`},`) + } + dst.Truncate(dst.Len() - 1) + // close both the records list, and the whole object. + _, _ = dst.WriteString(`}}`) + // In theory, there's no way for most of these to produce errors, + // but just in case, we'll check for an error every operation or so. + return dst.Err() +} + type errFieldNotFound struct { field string } diff --git a/ingest/codec_test.go b/ingest/codec_test.go index 4490d3729..8605eef5a 100644 --- a/ingest/codec_test.go +++ b/ingest/codec_test.go @@ -16,36 +16,481 @@ package ingest import ( "fmt" + "sort" + "strings" "testing" "time" "github.com/molecula/featurebase/v2/shardwidth" ) -func unusableSampleTranslator(keys ...string) (map[string]uint64, error) { - out := make(map[string]uint64, len(keys)) - for _, key := range keys { - out[key] = uint64(len(out)) * 13 +func TestStableTranslator(t *testing.T) { + tr := newStableTranslator() + m1, err := tr.TranslateKeys("a", "b") + if err != nil { + t.Fatalf("translation error on initial keys: %v", err) + } + m2, err := tr.TranslateIDs(m1["a"], m1["b"], 6) + if err != nil { + t.Fatalf("translation error on reverse lookup: %v", err) + } + m3, err := tr.TranslateKeys("a", "k-6") + if err != nil { + t.Fatalf("translation error on new keys: %v", err) + } + for k, v := range m3 { + if m2[v] != k { + t.Fatalf("expected round trip to equate %q and %d", k, v) + } } - return out, nil } -func TestSimpleCodec(t *testing.T) { - c, _ := NewJSONCodec(nil) - _ = c.AddSetField("set", nil) - _ = c.AddSetField("setkeys", unusableSampleTranslator) - _ = c.AddMutexField("mutex", nil) - _ = c.AddMutexField("mutexkeys", unusableSampleTranslator) - _ = c.AddTimeQuantumField("tq", nil) - _ = c.AddIntField("int", nil) - _ = c.AddIntField("intkeys", unusableSampleTranslator) +func TestMakeCodec(t *testing.T) { + codec, _ := NewJSONCodec(nil) + err := codec.AddSetField("set", nil) + if err != nil { + t.Fatalf("unexpected error creating field: %v", err) + } + err = codec.AddSetField("set", nil) + if err == nil { + t.Fatalf("expected error creating duplicate field, didn't get it") + } +} + +func TestEncode(t *testing.T) { + codec, _ := NewJSONCodec(nil) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) epoch, err := time.Parse("2006-01-02", "2020-01-01") if err != nil { t.Fatalf("can't parse sample epoch time: %v", err) } - _ = c.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) - _ = c.AddDecimalField("dec", 2) - _ = c.AddBoolField("bool") + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") + + codecs := []*JSONCodec{codec} + + // redo all of that, only on a keyed translator + codec, _ = NewJSONCodec(newStableTranslator()) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") + + codecs = append(codecs, codec) + + encodeTests := []*Request{ + { + Ops: []*Operation{ + { + OpType: OpWrite, + ClearRecordIDs: []uint64{0, 1, 2, 3, 5}, + ClearFields: []string{"bool", "dec", "int", "intkeys", "mutex", "mutexkeys", "set", "setkeys", "tq", "tqkeys", "ts"}, + FieldOps: map[string]*FieldOperation{ + "int": { + RecordIDs: []uint64{0, 1}, + Signed: []int64{1, -3}, + }, + "intkeys": { + RecordIDs: []uint64{0, 1}, + Signed: []int64{1, 1}, + }, + "set": { + RecordIDs: []uint64{0, 1, 2, 2}, + Values: []uint64{1, 1, 0, 1}, + }, + "setkeys": { + RecordIDs: []uint64{0, 1, 2, 2}, + Values: []uint64{1, 1, 0, 1}, + }, + "mutex": { + RecordIDs: []uint64{0, 1}, + Values: []uint64{1, 2}, + }, + "mutexkeys": { + RecordIDs: []uint64{0, 1}, + Values: []uint64{1, 2}, + }, + "tq": { + RecordIDs: []uint64{5, 5}, + Values: []uint64{8, 9}, + Signed: []int64{1234567890e9, 1234567890e9}, + }, + "tqkeys": { + RecordIDs: []uint64{3, 3}, + Values: []uint64{2, 4}, + Signed: []int64{1234567890e9, 1234567890e9}, + }, + "ts": { + RecordIDs: []uint64{0}, + Signed: []int64{1}, + }, + "bool": { + RecordIDs: []uint64{0, 1}, + Values: []uint64{0, 1}, + }, + "dec": { + RecordIDs: []uint64{0, 1, 2}, + Signed: []int64{123, -123, 0}, + }, + }, + }, + { + OpType: OpClear, + Seq: 1, + ClearRecordIDs: []uint64{6}, + ClearFields: []string{"tq"}, + }, + }, + }, + { + // this one needs to get filled in programmatically; see below + Ops: []*Operation{ + { + OpType: OpSet, + FieldOps: map[string]*FieldOperation{}, + }, + }, + }, + { + Ops: []*Operation{ + { + OpType: OpRemove, + FieldOps: map[string]*FieldOperation{ + "set": {}, + }, + }, + }, + }, + } + // and now we populate encodeTests[1] with a larger pool of data + const dataSize = 5000 + shardCount := uint64(600) // shards we want to target + passes := uint64(0) + recordIDs := make([]uint64, dataSize) + values := make([]uint64, dataSize) + timeStamps := make([]int64, dataSize) + signedValues := make([]int64, dataSize) + for i := uint64(0); i < dataSize; i++ { + if (i % shardCount) == 0 { + passes++ + } + recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes + values[i] = (i % 4) + timeStamps[i] = int64(1234567890e9 + (i * 100e9)) + signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys + } + // ensure record IDs are sorted, because other stuff might rely on this + sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) + op := encodeTests[1].Ops[0] + op.FieldOps["tq"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Values: append([]uint64{}, values...), + Signed: append([]int64{}, timeStamps...), + } + op.FieldOps["tqkeys"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Values: values, + Signed: timeStamps, + } + op.FieldOps["int"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Signed: append([]int64{}, signedValues...), + } + op.FieldOps["intkeys"] = &FieldOperation{ + RecordIDs: recordIDs, + Signed: signedValues, + } + // for sets, we want to shuffle things into fewer shards, and ensure + // non-duplication of values within each record, but also have lots + // of duplication of record IDs in the low shards + recordIDs = make([]uint64, dataSize) + values = make([]uint64, dataSize) + valuesPerRecord := uint64(5) + recordsPerShard := dataSize / valuesPerRecord / 30 + if recordsPerShard < 1 { + recordsPerShard = 1 + } + shard := uint64(0) + nextID := uint64(0) + nextValue := uint64(0) + for i := uint64(0); i < dataSize; i++ { + recordIDs[i] = nextID + values[i] = nextValue + (i % valuesPerRecord) + nextValue++ + if nextValue == valuesPerRecord { + nextValue = 0 + nextID++ + if nextID%(1< 1 { + valuesPerRecord-- + } + } + } + } + sort.Slice(recordIDs, func(i, j int) bool { return recordIDs[i] < recordIDs[j] }) + op.FieldOps["set"] = &FieldOperation{ + RecordIDs: append([]uint64{}, recordIDs...), + Values: append([]uint64{}, values...), + } + op.FieldOps["setkeys"] = &FieldOperation{ + RecordIDs: recordIDs, + Values: values, + } + var buf []byte + for i, tc := range encodeTests { + for _, c := range codecs { + data, err := c.AppendBytes(tc, buf[:0]) + if err != nil { + t.Fatalf("encode test %d: error encoding: %v", i, err) + } + // t.Logf("data:\n%s", data) + req, err := c.ParseBytes(data) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: error parsing: %v", i, err) + } + err = req.Compare(tc) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) + } + data, err = c.AppendBytes(req, buf[:0]) + if err != nil { + t.Fatalf("encode test %d: error encoding: %v", i, err) + } + // t.Logf("data:\n%s", data) + req2, err := c.ParseBytes(data) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: error parsing: %v", i, err) + } + err = req2.Compare(tc) + if err != nil { + t.Logf("encode test %d: data:\n%s", i, data) + t.Fatalf("encode test %d: round-trip mismatch: %v", i, err) + } + } + } +} + +func TestCodecErrors(t *testing.T) { + codec, _ := NewJSONCodec(nil) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddTimeQuantumField("tqkeys", newStableTranslator()) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) + epoch, err := time.Parse("2006-01-02", "2020-01-01") + if err != nil { + t.Fatalf("can't parse sample epoch time: %v", err) + } + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") + + testCases := []struct { + name string + json []byte + error string + }{ + { + name: "no action", + json: []byte(`[{"records":{"0":{"set":[0]}}}]`), + error: "action not specified", + }, + { + name: "unknown action", + json: []byte(`[{"action":"yeet","records":{"0":{"set":[0]}}}]`), + error: "unknown action", + }, + { + name: "unknown field", + json: []byte(`[{"action":"set","records":{"0":{"settee":[0]}}}]`), + error: "field not found", + }, + { + name: "unknown operation field", + json: []byte(`[{"action":"set","yeet":false,"records":{"0":{"set":[0]}}}]`), + error: "unknown operation field", + }, + { + name: "expected operation", + json: []byte(`[true]`), + error: "expected operation", + }, + { + name: "expecting key", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":0}}}]`), + error: "expecting key", + }, + { + name: "invalid int for bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":2}}}]`), + error: "boolean should be", + }, + { + name: "invalid number for bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":1.3}}}]`), + error: "looks like Number", + }, + { + name: "invalid string for bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":"truly"}}}]`), + error: "expecting boolean", + }, + { + name: "nonsense bool", + json: []byte(`[{"action":"set","records":{"0":{"bool":[]}}}]`), + error: "boolean should be", + }, + { + name: "expecting numeric value", + json: []byte(`[{"action":"set","records":{"0":{"set":0.1}}}]`), + error: "invalid syntax", + }, + { + name: "expecting value", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":true}}}]`), + error: "expecting value", + }, + { + name: "expecting array-key", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":[0]}}}]`), + error: "expecting key", + }, + { + name: "expecting array-value", + json: []byte(`[{"action":"set","records":{"0":{"setkeys":[true]}}}]`), + error: "expecting value", + }, + { + name: "expecting numeric array-value", + json: []byte(`[{"action":"set","records":{"0":{"set":[0.1]}}}]`), + error: "invalid syntax", + }, + { + name: "expecting numeric value", + json: []byte(`[{"action":"set","records":{"0":{"int":0.1}}}]`), + error: "invalid syntax", + }, + { + name: "expecting int key", + json: []byte(`[{"action":"set","records":{"0":{"intkeys":0}}}]`), + error: "expecting string key", + }, + { + name: "expecting int value", + json: []byte(`[{"action":"set","records":{"0":{"int":[0]}}}]`), + error: "expecting integer value", + }, + { + name: "expecting string array-value", + json: []byte(`[{"action":"set","records":{"0":{"intkeys":["a"]}}}]`), + error: "expecting string key", + }, + { + name: "expecting numeric mutex value", + json: []byte(`[{"action":"set","records":{"0":{"mutex":0.1}}}]`), + error: "invalid syntax", + }, + { + name: "expecting mutex key", + json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":0}}}]`), + error: "expecting string key", + }, + { + name: "expecting mutex value", + json: []byte(`[{"action":"set","records":{"0":{"mutex":[0]}}}]`), + error: "expecting integer value", + }, + { + name: "expecting mutex string value", + json: []byte(`[{"action":"set","records":{"0":{"mutexkeys":["a"]}}}]`), + error: "expecting string key", + }, + { + name: "time quantum invalid time", + json: []byte(`[{"action":"set","records":{"0":{"tq":{"time":[],"values":[3]}}}}]`), + error: "expecting time", + }, + { + name: "time stamp invalid integer", + json: []byte(`[{"action":"set","records":{"0":{"ts":1.3}}}]`), + error: "parsing numeric time", + }, + { + name: "time stamp invalid string", + json: []byte(`[{"action":"set","records":{"0":{"ts":"RFC3339"}}}]`), + error: "parsing time", + }, + { + name: "time stamp invalid type", + json: []byte(`[{"action":"set","records":{"0":{"ts":[]}}}]`), + error: "expecting time", + }, + { + name: "invalid decimal", + json: []byte(`[{"action":"set","records":{"0":{"dec":[]}}}]`), + error: "expecting floating", + }, + { + name: "duplicate record", + json: []byte(`[{"action":"set","records":{"0":{"int":1},"0":{"set":0}}}]`), + error: "duplicated in input", + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + req, err := codec.ParseBytes(tc.json) + if err == nil { + req.Dump(t.Logf) + t.Fatalf("expected error like %q, got request instead", tc.error) + } else { + msg := err.Error() + if !strings.Contains(msg, tc.error) { + t.Fatalf("expected error like %q, got %q", tc.error, msg) + } + } + }) + } +} + +func TestSimpleCodec(t *testing.T) { + codec, _ := NewJSONCodec(nil) + _ = codec.AddSetField("set", nil) + _ = codec.AddSetField("setkeys", newStableTranslator()) + _ = codec.AddMutexField("mutex", nil) + _ = codec.AddMutexField("mutexkeys", newStableTranslator()) + _ = codec.AddTimeQuantumField("tq", nil) + _ = codec.AddIntField("int", nil) + _ = codec.AddIntField("intkeys", newStableTranslator()) + epoch, err := time.Parse("2006-01-02", "2020-01-01") + if err != nil { + t.Fatalf("can't parse sample epoch time: %v", err) + } + _ = codec.AddTimestampField("ts", time.Millisecond, epoch.Unix()*1000) + _ = codec.AddDecimalField("dec", 2) + _ = codec.AddBoolField("bool") var nextShard = uint64(1< 0 { - subOp.Values = f.Values[prev:endIndex] - } - if len(f.Signed) > 0 { - subOp.Signed = f.Signed[prev:endIndex] - } - target[shard] = subOp - prev = endIndex +// clone makes a duplicate of the operation without shared storage +func (f *FieldOperation) clone() *FieldOperation { + f2 := &FieldOperation{ + RecordIDs: append([]uint64{}, f.RecordIDs...), + Values: append([]uint64{}, f.Values...), + Signed: append([]int64{}, f.Signed...), } + return f2 } func ShardIDs(ids []uint64) (out map[uint64][]uint64) { @@ -349,10 +396,22 @@ func (f *FieldOperation) SortByKeys(keys []uint64) { // doesn't help as much as you'd hope. This beats using stdlib sort by about // a factor of two for those small N, for larger N we're using the radix sort // that calls this. +// +// External keys exist only when we are sorting by value, which is to say, +// when we're using row-oriented formats (set, mutex, time quantum). +// For int/decimal/timestamp fields, we're sorting by record only. +// So, if keys is the same as f.RecordIDs, we're looking at an int field +// or equivalent, so Signed exists and Values doesn't exist. +// Otherwise, we might be looking at a time quantum field (both exist) +// or set/mutex (only Values exist). func simpleSort(f *FieldOperation, keys []uint64) { - if keys != nil { + // keys might actually just point to record IDs, in which case, we don't + // want to shuffle the corresponding RecordIDs too, because that would just + // reverse our swaps. If they're different, we actually need to swap them + // both. + if &keys[0] != &f.RecordIDs[0] { // sorting by record IDs - if f.Values != nil && f.Signed != nil { + if f.Values != nil && f.Signed != nil { // time quantum field for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -362,7 +421,7 @@ func simpleSort(f *FieldOperation, keys []uint64) { } } - } else if f.Values != nil { + } else if f.Values != nil { // set/mutex/bool for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -371,7 +430,7 @@ func simpleSort(f *FieldOperation, keys []uint64) { f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] } } - } else if f.Signed != nil { + } else if f.Signed != nil { // can't-happen, we think for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -379,7 +438,7 @@ func simpleSort(f *FieldOperation, keys []uint64) { f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] } } - } else { + } else { // can't-happen, we think for i := 1; i < len(keys); i++ { for j := i; j > 0 && keys[j-1] > keys[j]; j-- { keys[j-1], keys[j] = keys[j], keys[j-1] @@ -388,7 +447,14 @@ func simpleSort(f *FieldOperation, keys []uint64) { } } } else { - if f.Values != nil && f.Signed != nil { + if f.Values == nil && f.Signed != nil { + for i := 1; i < len(f.RecordIDs); i++ { + for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { + f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] + f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] + } + } + } else if f.Values != nil && f.Signed != nil { // can't happen, we think for i := 1; i < len(f.RecordIDs); i++ { for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] @@ -396,22 +462,14 @@ func simpleSort(f *FieldOperation, keys []uint64) { f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] } } - } else if f.Values != nil { + } else if f.Values != nil { // only happens during testing for i := 1; i < len(f.RecordIDs); i++ { for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1] } } - } else if f.Signed != nil { - for i := 1; i < len(f.RecordIDs); i++ { - for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { - f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] - f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1] - } - } - } else { - // why do we only have record IDs? I don't know + } else { // should definitely not happen for i := 1; i < len(f.RecordIDs); i++ { for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- { f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1] @@ -493,12 +551,7 @@ func sortPartialByKeys(f *FieldOperation, keys []uint64, shift int) { if end-start > 32 { sortPartialByKeys(&bucketOp, keys[start:end], nextShift) } else { - // naive stdlib sort - if externalKeys { - simpleSort(&bucketOp, keys[start:end]) - } else { - simpleSort(&bucketOp, nil) - } + simpleSort(&bucketOp, keys[start:end]) } } } @@ -534,16 +587,15 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error { if expected == nil { return nil } - if len(expected.RecordIDs) == 0 && len(expected.Values) == 0 && len(expected.Signed) == 0 { + // We don't worry about non-empty Values or Signed here, because in theory + // RecordIDs are the Source of Truth as to what's in the op. + if len(expected.RecordIDs) == 0 { return nil } return fmt.Errorf("expected field operation with %d records, got nil", len(expected.RecordIDs)) } if expected == nil { - if got == nil { - return nil - } - if len(got.RecordIDs) == 0 && len(got.Values) == 0 && len(got.Signed) == 0 { + if len(got.RecordIDs) == 0 { return nil } return fmt.Errorf("expected empty field operation, got %d records", len(got.RecordIDs)) @@ -578,52 +630,6 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error { return nil } -func translateUnsignedSlice(target []uint64, mapping []uint64) (err error) { - oops := 0 - for i, v := range target { - if v >= uint64(len(mapping)) { - oops++ - } else { - target[i] = mapping[v] - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range keys when applying translation mapping", oops) - } - return nil -} - -// TranslateUnsigned translates keys according to the provided mapping. This -// is used for sets, mutexes, and time quantums. -func (op *FieldOperation) TranslateKeys(mapping []uint64) error { - return translateUnsignedSlice(op.RecordIDs, mapping) -} - -// TranslateUnsigned translates values according to the provided mapping. This -// is used for sets, mutexes, and time quantums. -func (op *FieldOperation) TranslateUnsigned(mapping []uint64) error { - return translateUnsignedSlice(op.Values, mapping) -} - -// TranslateSigned translates signed values according to the provided mapping. -// If we're using this, it's because we're in an integer-type field, which -// admits using keys for fields, so all key values are actually non-negative, -// but the field's type still requires values be expressed as signed ints. -func (op *FieldOperation) TranslateSigned(mapping []uint64) error { - oops := 0 - for i, v := range op.Signed { - if v >= int64(len(mapping)) { - oops++ - } else { - op.Signed[i] = int64(mapping[v]) - } - } - if oops > 0 { - return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) - } - return nil -} - // ShardOperations is a set of Operations associated with a specific shard. type ShardOperations struct { Shard uint64 @@ -633,19 +639,17 @@ type ShardOperations struct { // Request is a complete ingest request, which may be any combination // of operations, which may apply to multiple shards. type Request struct { - FieldTypes map[string]FieldType - Ops []*Operation + Ops []*Operation } // ShardedRequest is an ingest request, split up into individual per-shard // operations. type ShardedRequest struct { - FieldTypes map[string]FieldType - Ops map[uint64][]*Operation + Ops map[uint64][]*Operation } // ByShard converts a request into the same request, only sharded. -func (r *Request) ByShard() (*ShardedRequest, error) { +func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error) { if len(r.Ops) == 0 { return &ShardedRequest{Ops: nil}, nil } @@ -662,12 +666,12 @@ func (r *Request) ByShard() (*ShardedRequest, error) { if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete { sharded := ShardIDs(op.ClearRecordIDs) for shard, data := range sharded { - shards[shard] = &Operation{OpType: op.OpType, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}} + shards[shard] = &Operation{OpType: op.OpType, Seq: op.Seq, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}} } } for field, fieldOp := range op.FieldOps { sharded := fieldOp.ByShard() - sorter := fieldTypeSorts[r.FieldTypes[field]] + sorter := fieldTypeSorts[fields[field]] if sorter == nil { sorter = (*FieldOperation).SortByRecords } @@ -678,7 +682,7 @@ func (r *Request) ByShard() (*ShardedRequest, error) { if op.OpType == OpWrite { return nil, fmt.Errorf("write operation has field operation data (%d items) for shard %d, but no clear data", len(data.RecordIDs), shard) } - shardOp = &Operation{OpType: op.OpType} + shardOp = &Operation{OpType: op.OpType, Seq: op.Seq} shards[shard] = shardOp shardOp.FieldOps = map[string]*FieldOperation{field: data} } else { @@ -697,18 +701,139 @@ func (r *Request) ByShard() (*ShardedRequest, error) { return &ShardedRequest{Ops: req}, nil } +// merge combines the components of a sharded request back into a single +// unsharded request, processing shards in numerical order. +func (s *ShardedRequest) merge() *Request { + req := &Request{} + if s == nil || len(s.Ops) == 0 { + return req + } + shards := make([]uint64, 0, len(s.Ops)) + for shard := range s.Ops { + shards = append(shards, shard) + } + sort.Slice(shards, func(i, j int) bool { return shards[i] < shards[j] }) + for _, shard := range shards { + ops := s.Ops[shard] + for _, op := range ops { + var _ *Operation + if op.Seq >= len(req.Ops) { + // Pad out with nil *Operations to the required length + req.Ops = append(req.Ops, make([]*Operation, op.Seq+1-len(req.Ops))...) + } + if req.Ops[op.Seq] == nil { + req.Ops[op.Seq] = op.clone() + continue + } + req.Ops[op.Seq].merge(op) + } + } + return req +} + func (r *Request) Dump(logf func(string, ...interface{})) { logf("req: %#v", r) for _, op := range r.Ops { logf("op: %#v", op) if len(op.ClearRecordIDs) > 0 { - logf(" clearRecordIDs: %d", op.ClearRecordIDs) + if len(op.ClearRecordIDs) > 8 { + logf(" clearRecordIDs: %d...+%d", op.ClearRecordIDs[:8], len(op.ClearRecordIDs)-8) + } else { + logf(" clearRecordIDs: %d", op.ClearRecordIDs) + } } if len(op.ClearFields) > 0 { - logf(" clearFields: %s", op.ClearFields) + if len(op.ClearFields) > 8 { + logf(" clearFields: %s...+%d", op.ClearFields[:8], len(op.ClearFields)-8) + } else { + logf(" clearFields: %s", op.ClearFields) + } } for field, fieldOp := range op.FieldOps { - logf(" field %q: %#v", field, fieldOp) + if fieldOp != nil { + logf(" field %q: op (%d/%d/%d)", field, len(fieldOp.RecordIDs), len(fieldOp.Values), len(fieldOp.Signed)) + if len(fieldOp.RecordIDs) > 0 { + if len(fieldOp.RecordIDs) > 8 { + logf(" records %d...+%d", fieldOp.RecordIDs[:8], len(fieldOp.RecordIDs)-8) + } else { + logf(" records %d", fieldOp.RecordIDs) + } + } + } else { + logf(" field %q: nil op", field) + } } } } + +func (r *Request) Compare(other *Request) error { + if other == nil { + if r != nil && len(r.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + if r == nil { + if other != nil && len(other.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + ops := r.Ops + ops2 := other.Ops + if len(ops2) != len(ops) { + return fmt.Errorf("expected %d ops, got %d", len(ops), len(ops2)) + } + for i, op := range ops { + if err := op.Compare(ops2[i]); err != nil { + return fmt.Errorf("op %d: %v", i, err) + } + } + return nil +} + +// Compare checks whether two ShardedRequest objects represent the same +// data. Empty shards shouldn't have entries in the map in the first place, +// so we don't accept a nil or 0-length slice of ops as equal to the +// shard key not existing, but we do accept nil or empty requests as +// equal to each other. +func (s *ShardedRequest) Compare(other *ShardedRequest) error { + if other == nil { + if s != nil && len(s.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + if s == nil { + if other != nil && len(other.Ops) != 0 { + return errors.New("non-empty sharded request can't equal empty/nil sharded request") + } + // empty and nil are allowed + return nil + } + for shard, ops := range s.Ops { + ops2, ok := other.Ops[shard] + if !ok { + return fmt.Errorf("shard %d missing in other", shard) + } + if len(ops2) != len(ops) { + return fmt.Errorf("shard %d: expected %d ops, got %d", shard, len(ops), len(ops2)) + } + for i, op := range ops { + if err := op.Compare(ops2[i]); err != nil { + return fmt.Errorf("shard %d, op %d: %v", shard, i, err) + } + } + } + if len(other.Ops) != len(s.Ops) { + for shard := range other.Ops { + if _, ok := s.Ops[shard]; !ok { + return fmt.Errorf("shard %d missing in self", shard) + } + } + } + return nil +} diff --git a/ingest/op_test.go b/ingest/op_test.go index b0b196e07..1646e8993 100644 --- a/ingest/op_test.go +++ b/ingest/op_test.go @@ -12,31 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ingest_test +package ingest import ( "math/rand" - "reflect" "testing" - "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/shardwidth" ) type opShardingTestCase struct { name string - input ingest.Request - output *ingest.ShardedRequest + input *Request + output *ShardedRequest } var opShardingTestCases = []opShardingTestCase{ { name: "sample", - input: ingest.Request{ - Ops: []*ingest.Operation{ + input: &Request{ + Ops: []*Operation{ { - OpType: ingest.OpSet, - FieldOps: map[string]*ingest.FieldOperation{ + OpType: OpSet, + FieldOps: map[string]*FieldOperation{ "shard0": { RecordIDs: []uint64{0, 1}, }, @@ -55,8 +53,9 @@ var opShardingTestCases = []opShardingTestCase{ }, }, { - OpType: ingest.OpRemove, - FieldOps: map[string]*ingest.FieldOperation{ + OpType: OpRemove, + Seq: 1, + FieldOps: map[string]*FieldOperation{ "shard0-2": { RecordIDs: []uint64{1, 2< 1 { + valuesPerRecord-- + } + } + } + } + op.FieldOps["set"] = &FieldOperation{ + RecordIDs: recordIDs, + Values: values, + } + fieldTypes := codec.FieldTypes() + sharded, err := req.ByShard(fieldTypes) + if err != nil { + t.Errorf("sharding: unexpected error %v", err) + } + merged := sharded.merge() + if err := req.Compare(merged); err != nil { + t.Fatalf("merge comparison: %v", err) } } @@ -139,7 +240,7 @@ func TestFancySharding(t *testing.T) { const recordCount = 5000 grr := rand.New(rand.NewSource(0)) for i := 0; i < 100; i++ { - f := &ingest.FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)} + f := &FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)} shards := make([]int, shardLimit) for j := range f.RecordIDs { v := uint64(grr.Int63n(shardLimit << shardwidth.Exponent)) diff --git a/ingest/translate_test.go b/ingest/translate_test.go new file mode 100644 index 000000000..67daf73c3 --- /dev/null +++ b/ingest/translate_test.go @@ -0,0 +1,137 @@ +// Copyright 2021 Molecula Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingest + +import ( + "fmt" + "testing" +) + +// stableTranslator implements a key translator that can be reused and +// will continue to give the same keys for the same values. Possibly +// surprisingly, it will invent new keys for IDs it is asked about but +// hasn't seen. This allows us to give a codec which would use keys on +// translation a request which contains arbitrary numbers, and request +// text that would parse into that request. +type stableTranslator struct { + in map[string]uint64 + out map[uint64]string + next uint64 +} + +func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + ret := make(map[string]uint64, len(keys)) + for _, key := range keys { + if existing, ok := s.in[key]; ok { + ret[key] = existing + continue + } + id := s.next + // but what if someone already translated that ID, so now it already + // exists? + if _, ok := s.out[id]; ok { + for k := range s.out { + if k > id { + id = k + } + } + // one larger than the largest we already have. this could + // wrap around, in which case, it's your own fault. + id++ + } + s.next = id + 1 + s.in[key] = id + s.out[id] = key + ret[key] = id + } + return ret, nil +} + +func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + ret := make(map[uint64]string, len(ids)) + for _, id := range ids { + if existing, ok := s.out[id]; ok { + ret[id] = existing + continue + } + key := fmt.Sprintf("k-%d", id) + s.in[key] = id + s.out[id] = key + ret[id] = key + if id >= s.next { + s.next = id + 1 + } + } + return ret, nil +} + +// newStableTranslator produces a translator which can translate forwards +// and backwards and invent new things if it needs to. Don't use this. +func newStableTranslator() *stableTranslator { + return &stableTranslator{ + in: make(map[string]uint64), + out: make(map[uint64]string), + } +} + +func TestTranslateReuse(t *testing.T) { + // the original stable-translator design had a flaw in that it + // assumed that each new ID would always come from a string translation, + // never from a key translation, and that they'd show up sequentially. + tr := newStableTranslator() + orig, err := tr.TranslateIDs(1, 2) + tr.next = 1 // intentionally break the translation logic for testing purposes + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + var s [6]string + stash := s[:0] + for _, v := range orig { + stash = append(stash, v) + } + for i := range s[len(orig):] { + stash = append(stash, fmt.Sprintf("key-%d", i)) + } + keys, err := tr.TranslateKeys(stash...) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + ids := make([]uint64, 0, len(keys)) + for _, v := range keys { + ids = append(ids, v) + } + idMap, err := tr.TranslateIDs(ids...) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + for k, v := range keys { + if idMap[v] != k { + t.Fatalf("translate mismatch: keys %q->%d, ids %d->%q", + k, v, v, idMap[v]) + } + } + for id, key := range idMap { + if keys[key] != id { + t.Fatalf("translate mismatch: ids %d->%q, keys %q->%d", + id, key, key, keys[key]) + } + } + for id, key := range orig { + if keys[key] != id { + t.Fatalf("translate mismatch: original ids %d->%q, keys %q->%d", + id, key, key, keys[key]) + } + } +} diff --git a/ingest/vec.go b/ingest/vec.go index 082ff8575..14d62cbbc 100644 --- a/ingest/vec.go +++ b/ingest/vec.go @@ -19,8 +19,6 @@ import ( "reflect" "strconv" "unsafe" - - "github.com/pkg/errors" ) // StringTable is a mapping of strings to temporary IDs. @@ -85,11 +83,8 @@ func (tbl *StringTable) IntID(in []byte) (int64, error) { // integers and a translation function from strings to "real" keys, yields // a translation/lookup slice. If it cannot translate all the keys, it // returns an error. -// -// The lookup function corresponds to the FindKeys/CreateKeys methods of -// featurebase translators, by an AMAZING coincidence. -func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint64, error)) ([]uint64, error) { - lookedUp, err := lookup(tbl.names...) +func (tbl *StringTable) MakeIDMap(keys KeyTranslator) ([]uint64, error) { + lookedUp, err := keys.TranslateKeys(tbl.names...) if err != nil { return nil, err } @@ -103,22 +98,38 @@ func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint return out, nil } -// TimeFormatForUnit returns the time transfer format (between the update encoder and the update applier) with appropriate resolution for a quantum unit. -func TimeFormatForUnit(unit rune) string { - switch unit { - case 'Y': - return "2006" - case 'M': - return "200601" - case 'D': - return "20060102" - case 'H': - return "2006010203" - default: - panic(errors.Errorf("invalid quantum unit: %q", unit)) +// translateSigned replaces values from 0 to len(mapping)-1 with the +// elements of mapping. It yields an error if any values aren't +// mapped. +func translateSigned(mapping []uint64, values []int64) error { + oops := 0 + for i, v := range values { + if v >= int64(len(mapping)) { + oops++ + } else { + values[i] = int64(mapping[v]) + } } + if oops > 0 { + return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) + } + return nil } -// TODO: bool - -// TODO: timestamp (just sugar on top of IntVector) +// translateUnsigned replaces values from 0 to len(mapping)-1 with the +// elements of mapping. It yields an error if any values aren't +// mapped. +func translateUnsigned(mapping []uint64, values []uint64) error { + oops := 0 + for i, v := range values { + if v >= uint64(len(mapping)) { + oops++ + } else { + values[i] = mapping[v] + } + } + if oops > 0 { + return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops) + } + return nil +} diff --git a/ingest/vec_test.go b/ingest/vec_test.go new file mode 100644 index 000000000..45a103645 --- /dev/null +++ b/ingest/vec_test.go @@ -0,0 +1,114 @@ +// Copyright 2021 Molecula Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package ingest + +import ( + "errors" + "testing" +) + +type badTranslator struct{} + +func (b badTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + if len(keys) == 0 { + return nil, errors.New("no keys") + } + m := make(map[string]uint64) + skip := true + for i, k := range keys { + if skip { + skip = false + continue + } + m[k] = uint64(i) + } + out := make([]uint64, len(keys)-1) + for i := range out { + out[i] = uint64(i) + } + return nil, nil +} + +func (b badTranslator) TranslateIDs(...uint64) (map[uint64]string, error) { + return nil, nil +} + +func TestStringTableErrors(t *testing.T) { + tbl := NewStringTable() + btr := badTranslator{} + _, keyErr := tbl.MakeIDMap(btr) + if keyErr == nil { + t.Fatalf("expected error passed up from failed translate, didn't get it") + } + a1, err := tbl.ID([]byte("a")) + if err != nil { + t.Fatalf("getting translation for key: %v", err) + } + b1, err := tbl.ID([]byte("b")) + if err != nil { + t.Fatalf("getting translation for key: %v", err) + } + _, keyErr = tbl.MakeIDMap(btr) + if keyErr == nil { + t.Fatalf("expected error for short translate, didn't get it") + } + tr := newStableTranslator() + _, err = tr.TranslateKeys("c", "d") + if err != nil { + t.Fatalf("translating stray keys: %v", err) + } + m, err := tbl.MakeIDMap(tr) + if err != nil { + t.Fatalf("creating lookup: %v", err) + } + var y = []uint64{a1, b1} + err = translateUnsigned(m, y) + if err != nil { + t.Fatalf("unexpected unsigned translation error: %v", err) + } + trResults, err := tr.TranslateKeys("a", "b") + if err != nil { + t.Fatalf("unexpected translation error: %v", err) + } + if y[0] != trResults["a"] { + t.Fatalf("expected %d, got %d", trResults["a"], y[0]) + } + if y[1] != trResults["b"] { + t.Fatalf("expected %d, got %d", trResults["b"], y[1]) + } + y[0] = a1 + y[1] = (a1 + b1 + 1) // assumed not to be any of them + err = translateUnsigned(m, y) + if err == nil { + t.Fatalf("no error from translating invalid table") + } + z := []int64{int64(a1), int64(b1)} + err = translateSigned(m, z) + if err != nil { + t.Fatalf("unexpected unsigned translation error: %v", err) + } + if uint64(z[0]) != trResults["a"] { + t.Fatalf("expected %d, got %d", trResults["a"], z[0]) + } + if uint64(z[1]) != trResults["b"] { + t.Fatalf("expected %d, got %d", trResults["b"], z[1]) + } + z[0] = int64(a1) + z[1] = int64(a1 + b1 + 1) // assumed not to be any of them + err = translateSigned(m, z) + if err == nil { + t.Fatalf("no error from translating invalid table") + } +} diff --git a/ingest_test.go b/ingest_test.go index 649f5dd16..5cd4beac4 100644 --- a/ingest_test.go +++ b/ingest_test.go @@ -26,9 +26,7 @@ import ( "testing" pilosa "github.com/molecula/featurebase/v2" - "github.com/molecula/featurebase/v2/http" "github.com/molecula/featurebase/v2/ingest" - "github.com/molecula/featurebase/v2/server" "github.com/molecula/featurebase/v2/test" "github.com/pkg/errors" ) @@ -448,14 +446,7 @@ func TestIngestTestcases(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) defer cancel() - c := test.MustRunCluster(t, 1, - []server.CommandOption{ - server.OptCommandServerOptions( - pilosa.OptServerNodeID("node0"), - pilosa.OptServerClusterHasher(&offsetModHasher{}), - pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), - )}, - ) + c := test.MustRunCluster(t, 3) defer c.Close() coord := c.GetPrimary() diff --git a/internal/clustertests/cluster_test.go b/internal/clustertests/cluster_test.go index 892e66383..3726bcc3b 100644 --- a/internal/clustertests/cluster_test.go +++ b/internal/clustertests/cluster_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/disco" picli "github.com/molecula/featurebase/v2/http" ) @@ -115,7 +115,7 @@ func waitForStatus(t *testing.T, stator func(context.Context) (string, error), s if err != nil { t.Logf("Status (try %d/%d): %v (retrying in %s)", i, n, err, sleep.String()) } else { - t.Logf("Status (try %d/%d): %s (retrying in %s)", i, n, s, sleep.String()) + t.Logf("Status (try %d/%d): curr: %s, expect: %s (retrying in %s)", i, n, s, status, sleep.String()) } if s == status { return diff --git a/internal/clustertests/docker-compose.yml b/internal/clustertests/docker-compose.yml index 0c1bf33c1..4192be6f8 100644 --- a/internal/clustertests/docker-compose.yml +++ b/internal/clustertests/docker-compose.yml @@ -14,6 +14,7 @@ services: - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301 - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301 + - PILOSA_CLUSTER_REPLICAS=3 networks: - pilosanet command: @@ -32,6 +33,7 @@ services: - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301 - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301 + - PILOSA_CLUSTER_REPLICAS=3 networks: - pilosanet command: @@ -50,6 +52,7 @@ services: - PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301 - PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301 - PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301 + - PILOSA_CLUSTER_REPLICAS=3 networks: - pilosanet command: @@ -57,6 +60,10 @@ services: client1: build: context: . + depends_on: + - "pilosa1" + - "pilosa2" + - "pilosa3" environment: - ENABLE_PILOSA_CLUSTER_TESTS=1 - GO111MODULE=on diff --git a/internal/clustertests/pause_node_test.go b/internal/clustertests/pause_node_test.go new file mode 100644 index 000000000..2de330a74 --- /dev/null +++ b/internal/clustertests/pause_node_test.go @@ -0,0 +1,412 @@ +// Copyright 2017 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clustertest + +import ( + "context" + "fmt" + "hash/fnv" + "io/ioutil" + "math/rand" + "os" + "os/exec" + "path/filepath" + "strconv" + "testing" + "time" + + pilosa "github.com/molecula/featurebase/v2" + boltdb "github.com/molecula/featurebase/v2/boltdb" + "github.com/molecula/featurebase/v2/disco" + "github.com/molecula/featurebase/v2/http" + picli "github.com/molecula/featurebase/v2/http" + "github.com/molecula/featurebase/v2/net" + "github.com/molecula/featurebase/v2/topology" + "github.com/pkg/errors" +) + +func sendCmd(cmd string, args ...string) error { + pcmd := exec.Command(cmd, args...) + pcmd.Stdout = os.Stdout + pcmd.Stderr = os.Stderr + err := pcmd.Start() + if err != nil { + return errors.Wrap(err, "starting cmd") + } + err = pcmd.Wait() + if err != nil { + return errors.Wrap(err, "waiting on cmd") + } + return nil +} + +func unpauseNode(node string) error { + unpauseArgs := []string{"container", "unpause", "clustertests_" + node + "_1"} + return sendCmd("docker", unpauseArgs...) +} + +func pauseNode(node string) error { + pauseArgs := []string{"container", "pause", "clustertests_" + node + "_1"} + return sendCmd("docker", pauseArgs...) +} + +type keyInserter struct { + client *http.InternalClient + uri *net.URI + index string + keys []string +} + +func (ki keyInserter) insertKeys(ctx context.Context) (map[string]uint64, error) { + ts, err := ki.client.CreateIndexKeysNode(ctx, ki.uri, ki.index, ki.keys...) + return ts, err +} + +func getAddress(node string) string { + return node + ":10101" +} + +func getClients(addrs []string) ([]*http.InternalClient, error) { + clients := make([]*http.InternalClient, 0, len(addrs)) + for _, addr := range addrs { + c, err := picli.NewInternalClient(addr, picli.GetHTTPClient(nil)) + if err != nil { + return nil, err + } + clients = append(clients, c) + } + return clients, nil +} + +func getURIsFromAddresses(addrs []string) ([]*net.URI, error) { + uris := make([]*net.URI, 0, len(addrs)) + for _, addr := range addrs { + uri, err := net.NewURIFromAddress(addr) + if err != nil { + return nil, err + } + uris = append(uris, uri) + } + return uris, nil +} + +func readIndexTranslateData(ctx context.Context, client *picli.InternalClient, dirPath, index string, partition int) error { + // read translateStore contents from endpoint + r, err := client.IndexTranslateDataReader(ctx, index, partition) + if err != nil { + return err + } + buf, err := ioutil.ReadAll(r) + if err != nil { + return err + } + r.Close() + + // create file and write contents to it + filename := strconv.FormatInt(int64(partition), 10) + filePath := filepath.Join(dirPath, filename) + file, err := os.Create(filePath) + if err != nil { + return err + } + _, err = file.Write(buf) + if err != nil { + file.Close() + return err + } + err = file.Sync() + if err != nil { + file.Close() + return err + } + file.Close() + return nil +} + +func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, error) { + dirEntries, err := ioutil.ReadDir(dirPath) + if err != nil { + return nil, err + } + + // in case of error, close any translateStore that has been opened + rollback := make([]pilosa.TranslateStore, 0, topology.DefaultPartitionN) + defer func() { + for _, ts := range rollback { + _ = ts.Close() + } + }() + + // filter out non-file entries + filePaths := make([]string, 0, len(dirEntries)) + for _, entry := range dirEntries { + if entry.Mode().IsDir() { + continue + } + filePath := filepath.Join(dirPath, entry.Name()) + filePaths = append(filePaths, filePath) + } + + translateStores := make(map[int]pilosa.TranslateStore) + for _, filePath := range filePaths { + // extract partition number from filename + file := filepath.Base(filePath) + partition, err := strconv.Atoi(file) + if err != nil { + return nil, err + } + // open bolt db + ts, err := boltdb.OpenTranslateStore(filePath, index, "", partition, topology.DefaultPartitionN, false) + ts.SetReadOnly(true) + if err != nil { + return nil, err + } + translateStores[partition] = ts + rollback = append(rollback, ts) + } + rollback = nil + + return translateStores, nil +} + +var errOpRetriable = errors.New("If operation failed on this error, it can be retried") + +func verifyNodeHasGivenKeys(ctx context.Context, node, index, dirPath string, keys []string) error { + // get client that's connected to node + address := getAddress(node) + client, err := picli.NewInternalClient(address, picli.GetHTTPClient(nil)) + if err != nil { + return err + } + + // create dir to store boltdbs for this node + nodeDirPath := filepath.Join(dirPath, node) + err = os.Mkdir(nodeDirPath, 0755) + if err != nil { + return err + } + + // read in all the translate stores for each partition + for partition := 0; partition < topology.DefaultPartitionN; partition++ { + err := readIndexTranslateData(ctx, client, nodeDirPath, index, partition) + if err != nil { + return err + } + } + + // open all the translate stores + translateStores, err := openTranslateStores(nodeDirPath, index) + if err != nil { + return err + } + + // close all the translate stores on complete + defer func() { + for _, ts := range translateStores { + ts.Close() + } + }() + + // merge all translations + merged := make(map[string]uint64) + for _, ts := range translateStores { + entries, err := ts.FindKeys(keys...) + if err != nil { + return err + } + for key, id := range entries { + merged[key] = id + } + } + + // check that all expected keys present in node + if len(merged) != len(keys) { + msg := fmt.Sprintf("entries in node %s: %d. keys inserted: %d", + node, len(merged), len(keys)) + return errors.Wrap(errOpRetriable, msg) + } + for _, k := range keys { + if _, ok := merged[k]; !ok { + msg := fmt.Sprintf("Key '%s' not present in node %s", k, node) + return errors.Wrap(errOpRetriable, msg) + } + } + + return nil +} + +func genKeys(count, maxTries int, keyToNode func(string) string, filterOut []string) []string { + exclusionSet := make(map[string]struct{}) + for _, n := range filterOut { + exclusionSet[n] = struct{}{} + } + keys := make([]string, 0, count) + i := 0 + for { + if i >= maxTries { + break + } + key := fmt.Sprintf("key-%d", i) + i++ + // get primary node for this key + node := keyToNode(key) + // check if we should exclude this key + if _, ok := exclusionSet[node]; ok { + continue + } + // add key + keys = append(keys, key) + if len(keys) >= count { + return keys + } + } + return keys +} + +func TestPauseReplica(t *testing.T) { + if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" { + t.Skip("pilosa cluster tests for replication when a replica is paused are not enabled") + } + // configurations for test + nodeNames := []string{"pilosa1", "pilosa2", "pilosa3"} + nodeToPause := "pilosa3" + addresses := make([]string, len(nodeNames)) + for i, node := range nodeNames { + addresses[i] = getAddress(node) + } + clients, err := getClients(addresses) + if err != nil { + t.Fatalf("on init clients from addresses: %v, %v", addresses, err) + } + cli := clients[0] + uris, err := getURIsFromAddresses(addresses) + if err != nil { + t.Fatalf("on init clients from addresses: %v, %v", addresses, err) + } + uri := uris[0] + + ctx := context.Background() + ctx, cancel := context.WithCancel(ctx) + + t.Log("start Client") + + // first achieve normal cluster status + waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second) + + // create keyed index + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + index := fmt.Sprintf("keyed-index-%d", rng.Int63()) + err = cli.EnsureIndex(ctx, index, pilosa.IndexOptions{ + Keys: true, + }) + if err != nil { + t.Fatalf("creating/asserting index: %v", err) + } + + // generate mapping from partition to primary node + partitionToNode := make([]string, topology.DefaultPartitionN) + for partition := 0; partition < topology.DefaultPartitionN; partition++ { + nodes, err := cli.PartitionNodes(ctx, partition) + if err != nil { + t.Fatal(err) + } + partitionToNode[partition] = nodes[0].URI.Host + } + + // generate random keys to insert + // no guarantee that all keys expected to be generated don't fall + // in nodes to be filtered out + keyCount := 100 + maxKeyGenTries := 1000 + filterOutKeysFromTheseNodes := []string{nodeToPause} + keyToNode := func(key string) string { + // get partition for this key + h := fnv.New64a() + _, _ = h.Write([]byte(index)) + _, _ = h.Write([]byte(key)) + partition := int(h.Sum64() % uint64(topology.DefaultPartitionN)) + // get node for this partition + return partitionToNode[partition] + } + keys := genKeys(keyCount, maxKeyGenTries, keyToNode, filterOutKeysFromTheseNodes) + + // pause node + t.Logf("pause %s", nodeToPause) + err = pauseNode(nodeToPause) + if err != nil { + t.Fatalf("error on pause node %s: %v", nodeToPause, err) + } + + // insert keys + t.Log("start insert") + ts, err := keyInserter{ + client: cli, + uri: uri, + index: index, + keys: keys, + }.insertKeys(ctx) + if err != nil { + t.Fatalf("Error: inserting index keys for index(%s) send to node(%s): %v", index, uri.String(), err) + } + t.Logf("successfully end insert: %v", len(ts)) + + // wait for cluster status to be non-normal + waitForStatus(t, cli.Status, string(disco.ClusterStateDegraded), 30, 1*time.Second) + + // wait for cluster status to get back to normal + t.Logf("unpause %s", nodeToPause) + err = unpauseNode(nodeToPause) + if err != nil { + t.Fatalf("error on unpause node %s: %v", nodeToPause, err) + } + waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second) + + // set up directory to store keys + basePath := "." + keysDirName := "index_keys" + dirPath, err := filepath.Abs(basePath) + if err != nil { + t.Fatal(err) + } + dirPath = filepath.Join(dirPath, keysDirName) + err = os.Mkdir(dirPath, 0755) + if err != nil { + t.Fatal(err) + } + + maxRetries := 10 + durationInBetweenRetries := 5 * time.Second + for _, node := range nodeNames { + try := 1 + retries: + for { + err = verifyNodeHasGivenKeys(ctx, node, index, dirPath, keys) + if err == nil { + break retries + } + if try <= maxRetries && errors.Is(err, errOpRetriable) { + try++ + t.Logf("node %s, retry verify key replication: (%d/%d) after %v\n", + node, try, maxRetries, durationInBetweenRetries) + time.Sleep(durationInBetweenRetries) + continue + } + t.Fatal(errors.Wrap(err, fmt.Sprintf("try: (%d/%d)", try, maxRetries))) + } + } + + cancel() + t.Log("Done") +} diff --git a/pb/private.pb.go b/pb/private.pb.go index 54366c2cb..3a2807420 100644 --- a/pb/private.pb.go +++ b/pb/private.pb.go @@ -2479,6 +2479,234 @@ func (m *ResizeNodeMessage) GetAction() string { return "" } +type FieldOperation struct { + RecordIDs []uint64 `protobuf:"varint,1,rep,packed,name=RecordIDs,proto3" json:"RecordIDs,omitempty"` + Values []uint64 `protobuf:"varint,2,rep,packed,name=Values,proto3" json:"Values,omitempty"` + Signed []int64 `protobuf:"varint,3,rep,packed,name=Signed,proto3" json:"Signed,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *FieldOperation) Reset() { *m = FieldOperation{} } +func (m *FieldOperation) String() string { return proto.CompactTextString(m) } +func (*FieldOperation) ProtoMessage() {} +func (*FieldOperation) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{39} +} +func (m *FieldOperation) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *FieldOperation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_FieldOperation.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *FieldOperation) XXX_Merge(src proto.Message) { + xxx_messageInfo_FieldOperation.Merge(m, src) +} +func (m *FieldOperation) XXX_Size() int { + return m.Size() +} +func (m *FieldOperation) XXX_DiscardUnknown() { + xxx_messageInfo_FieldOperation.DiscardUnknown(m) +} + +var xxx_messageInfo_FieldOperation proto.InternalMessageInfo + +func (m *FieldOperation) GetRecordIDs() []uint64 { + if m != nil { + return m.RecordIDs + } + return nil +} + +func (m *FieldOperation) GetValues() []uint64 { + if m != nil { + return m.Values + } + return nil +} + +func (m *FieldOperation) GetSigned() []int64 { + if m != nil { + return m.Signed + } + return nil +} + +type ShardIngestOperation struct { + OpType string `protobuf:"bytes,1,opt,name=OpType,proto3" json:"OpType,omitempty"` + ClearRecordIDs []uint64 `protobuf:"varint,2,rep,packed,name=ClearRecordIDs,proto3" json:"ClearRecordIDs,omitempty"` + ClearFields []string `protobuf:"bytes,3,rep,name=ClearFields,proto3" json:"ClearFields,omitempty"` + FieldOps map[string]*FieldOperation `protobuf:"bytes,4,rep,name=FieldOps,proto3" json:"FieldOps,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardIngestOperation) Reset() { *m = ShardIngestOperation{} } +func (m *ShardIngestOperation) String() string { return proto.CompactTextString(m) } +func (*ShardIngestOperation) ProtoMessage() {} +func (*ShardIngestOperation) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{40} +} +func (m *ShardIngestOperation) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardIngestOperation) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardIngestOperation.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardIngestOperation) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardIngestOperation.Merge(m, src) +} +func (m *ShardIngestOperation) XXX_Size() int { + return m.Size() +} +func (m *ShardIngestOperation) XXX_DiscardUnknown() { + xxx_messageInfo_ShardIngestOperation.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardIngestOperation proto.InternalMessageInfo + +func (m *ShardIngestOperation) GetOpType() string { + if m != nil { + return m.OpType + } + return "" +} + +func (m *ShardIngestOperation) GetClearRecordIDs() []uint64 { + if m != nil { + return m.ClearRecordIDs + } + return nil +} + +func (m *ShardIngestOperation) GetClearFields() []string { + if m != nil { + return m.ClearFields + } + return nil +} + +func (m *ShardIngestOperation) GetFieldOps() map[string]*FieldOperation { + if m != nil { + return m.FieldOps + } + return nil +} + +type ShardIngestOperations struct { + Ops []*ShardIngestOperation `protobuf:"bytes,1,rep,name=Ops,proto3" json:"Ops,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardIngestOperations) Reset() { *m = ShardIngestOperations{} } +func (m *ShardIngestOperations) String() string { return proto.CompactTextString(m) } +func (*ShardIngestOperations) ProtoMessage() {} +func (*ShardIngestOperations) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{41} +} +func (m *ShardIngestOperations) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardIngestOperations) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardIngestOperations.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardIngestOperations) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardIngestOperations.Merge(m, src) +} +func (m *ShardIngestOperations) XXX_Size() int { + return m.Size() +} +func (m *ShardIngestOperations) XXX_DiscardUnknown() { + xxx_messageInfo_ShardIngestOperations.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardIngestOperations proto.InternalMessageInfo + +func (m *ShardIngestOperations) GetOps() []*ShardIngestOperation { + if m != nil { + return m.Ops + } + return nil +} + +type ShardedIngestRequest struct { + Ops map[uint64]*ShardIngestOperations `protobuf:"bytes,1,rep,name=Ops,proto3" json:"Ops,omitempty" protobuf_key:"varint,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ShardedIngestRequest) Reset() { *m = ShardedIngestRequest{} } +func (m *ShardedIngestRequest) String() string { return proto.CompactTextString(m) } +func (*ShardedIngestRequest) ProtoMessage() {} +func (*ShardedIngestRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_d2a91b51c7bdc125, []int{42} +} +func (m *ShardedIngestRequest) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ShardedIngestRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ShardedIngestRequest.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ShardedIngestRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ShardedIngestRequest.Merge(m, src) +} +func (m *ShardedIngestRequest) XXX_Size() int { + return m.Size() +} +func (m *ShardedIngestRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ShardedIngestRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ShardedIngestRequest proto.InternalMessageInfo + +func (m *ShardedIngestRequest) GetOps() map[uint64]*ShardIngestOperations { + if m != nil { + return m.Ops + } + return nil +} + func init() { proto.RegisterType((*IndexMeta)(nil), "pb.IndexMeta") proto.RegisterType((*FieldOptions)(nil), "pb.FieldOptions") @@ -2520,103 +2748,121 @@ func init() { proto.RegisterType((*TransactionStats)(nil), "pb.TransactionStats") proto.RegisterType((*ResizeAbortMessage)(nil), "pb.ResizeAbortMessage") proto.RegisterType((*ResizeNodeMessage)(nil), "pb.ResizeNodeMessage") + proto.RegisterType((*FieldOperation)(nil), "pb.FieldOperation") + proto.RegisterType((*ShardIngestOperation)(nil), "pb.ShardIngestOperation") + proto.RegisterMapType((map[string]*FieldOperation)(nil), "pb.ShardIngestOperation.FieldOpsEntry") + proto.RegisterType((*ShardIngestOperations)(nil), "pb.ShardIngestOperations") + proto.RegisterType((*ShardedIngestRequest)(nil), "pb.ShardedIngestRequest") + proto.RegisterMapType((map[uint64]*ShardIngestOperations)(nil), "pb.ShardedIngestRequest.OpsEntry") } func init() { proto.RegisterFile("private.proto", fileDescriptor_d2a91b51c7bdc125) } var fileDescriptor_d2a91b51c7bdc125 = []byte{ - // 1450 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x57, 0x5d, 0x6f, 0x1b, 0x45, - 0x17, 0x7e, 0xf7, 0xc3, 0xb1, 0x7d, 0x1c, 0x27, 0xce, 0x34, 0xea, 0xbb, 0xfd, 0x78, 0x23, 0x77, - 0x5e, 0x44, 0x43, 0x25, 0x22, 0x51, 0x2e, 0x8a, 0xe0, 0xa6, 0x49, 0x9c, 0x16, 0x53, 0xd2, 0x86, - 0x71, 0xda, 0x5b, 0x34, 0x5e, 0x8f, 0x9a, 0x55, 0xd6, 0xbb, 0x66, 0x3f, 0x52, 0xbb, 0x17, 0x48, - 0x20, 0x10, 0xfc, 0x04, 0x7e, 0x06, 0x37, 0xfc, 0x07, 0x6e, 0x90, 0xf8, 0x09, 0xa8, 0xfc, 0x11, - 0x34, 0x67, 0x66, 0x76, 0xd7, 0xae, 0x5b, 0x43, 0xc4, 0xdd, 0x9e, 0xe7, 0xcc, 0x9c, 0xf3, 0x9c, - 0x8f, 0x39, 0x33, 0x0b, 0xed, 0x49, 0x12, 0x5c, 0xf0, 0x4c, 0xec, 0x4d, 0x92, 0x38, 0x8b, 0x89, - 0x3d, 0x19, 0x5e, 0x5f, 0x9f, 0xe4, 0xc3, 0x30, 0xf0, 0x15, 0x42, 0x1f, 0x42, 0xb3, 0x1f, 0x8d, - 0xc4, 0xf4, 0x58, 0x64, 0x9c, 0x10, 0x70, 0x1f, 0x89, 0x59, 0xea, 0x39, 0x5d, 0x6b, 0xb7, 0xc1, - 0xf0, 0x9b, 0xbc, 0x0b, 0x1b, 0xa7, 0x09, 0xf7, 0xcf, 0x8f, 0xa6, 0x41, 0x9a, 0x89, 0xc8, 0x17, - 0x9e, 0x8b, 0xda, 0x05, 0x94, 0xfe, 0xec, 0xc0, 0xfa, 0x83, 0x40, 0x84, 0xa3, 0x27, 0x93, 0x2c, - 0x88, 0xa3, 0x54, 0x1a, 0x3b, 0x9d, 0x4d, 0x84, 0xd7, 0xe8, 0x5a, 0xbb, 0x4d, 0x86, 0xdf, 0xe4, - 0x26, 0x34, 0x0f, 0xb9, 0x7f, 0x26, 0x50, 0xe1, 0xa0, 0xa2, 0x04, 0x0a, 0xed, 0x20, 0x78, 0xa9, - 0xbc, 0xb4, 0x59, 0x09, 0x90, 0x2e, 0xb4, 0x4e, 0x83, 0xb1, 0xf8, 0x22, 0xe7, 0x51, 0x96, 0x8f, - 0xbd, 0x1a, 0xee, 0xae, 0x42, 0xe4, 0x2a, 0xac, 0x3d, 0x09, 0x47, 0xc7, 0x41, 0xe4, 0x35, 0xbb, - 0xd6, 0xae, 0xc3, 0xb4, 0x64, 0x70, 0x3e, 0xf5, 0xa0, 0xc4, 0xf9, 0xb4, 0x08, 0xb7, 0x35, 0x1f, - 0xee, 0xe3, 0x78, 0x90, 0xf1, 0x68, 0xc4, 0x93, 0xd1, 0xb3, 0x40, 0xbc, 0xf0, 0xd6, 0x55, 0xb8, - 0xf3, 0xa8, 0xdc, 0x7b, 0xc0, 0x53, 0xe1, 0xb5, 0xd1, 0x22, 0x7e, 0x93, 0xeb, 0xd0, 0x38, 0x08, - 0xb2, 0x9e, 0x98, 0x64, 0x67, 0xde, 0x46, 0xd7, 0xda, 0x75, 0x59, 0x21, 0x93, 0x6d, 0xa8, 0x0d, - 0x7c, 0x1e, 0x0a, 0x6f, 0x13, 0x37, 0x28, 0x81, 0x50, 0x58, 0x7f, 0x10, 0x27, 0x22, 0x78, 0x1e, - 0x61, 0x11, 0xbc, 0x0e, 0x06, 0x35, 0x87, 0x91, 0xff, 0x81, 0x23, 0x43, 0xda, 0xea, 0x5a, 0xbb, - 0xad, 0xbb, 0xad, 0xbd, 0xc9, 0x70, 0xaf, 0x27, 0xfc, 0x60, 0xcc, 0x43, 0x26, 0x71, 0x54, 0xf3, - 0xa9, 0x47, 0x96, 0xa9, 0xf9, 0x54, 0x72, 0x92, 0x29, 0x7a, 0x1a, 0x05, 0x99, 0x77, 0x05, 0xad, - 0x17, 0x32, 0xa5, 0xb0, 0xd1, 0x1f, 0x4f, 0xe2, 0x24, 0x63, 0x22, 0x9d, 0xc4, 0x51, 0x2a, 0x48, - 0x07, 0x9c, 0xa3, 0x24, 0xf1, 0x2c, 0x5c, 0x28, 0x3f, 0xe9, 0xd7, 0xd0, 0x39, 0x08, 0x63, 0xff, - 0xbc, 0xc7, 0x33, 0xce, 0xc4, 0x57, 0xb9, 0x48, 0x33, 0x19, 0x8b, 0xa2, 0xab, 0xd6, 0x29, 0x41, - 0xa2, 0x58, 0x7f, 0xcf, 0x56, 0x28, 0x0a, 0x32, 0x4f, 0x98, 0x45, 0x55, 0x2e, 0xfc, 0xc6, 0x5c, - 0x9c, 0xf1, 0x64, 0x84, 0x35, 0x76, 0x99, 0x12, 0x24, 0x8a, 0x9e, 0xb0, 0x2f, 0x5c, 0xa6, 0x04, - 0xda, 0x87, 0xad, 0x8a, 0x7f, 0x4d, 0xf3, 0x2a, 0xac, 0xb1, 0xf8, 0x45, 0xbf, 0x97, 0x7a, 0x56, - 0xd7, 0xd9, 0x75, 0x99, 0x96, 0xb0, 0x81, 0xe2, 0x30, 0x1f, 0x47, 0x52, 0x65, 0xa3, 0xaa, 0x04, - 0xe8, 0x35, 0xa8, 0x61, 0x37, 0xc9, 0x28, 0xcb, 0xbd, 0xf2, 0x93, 0x7e, 0x63, 0x41, 0xf3, 0x98, - 0x4f, 0x91, 0x48, 0x4a, 0xee, 0x41, 0xc3, 0xd4, 0x1a, 0x17, 0xb5, 0xee, 0xde, 0x90, 0x79, 0x2d, - 0x16, 0xec, 0x19, 0xed, 0x51, 0x94, 0x25, 0x33, 0x56, 0x2c, 0xbe, 0xfe, 0x09, 0xb4, 0xe7, 0x54, - 0xd2, 0xd3, 0xb9, 0x98, 0x99, 0x7c, 0x9e, 0x8b, 0x99, 0x8c, 0xf2, 0x82, 0x87, 0xb9, 0xc0, 0x2c, - 0xb9, 0x4c, 0x09, 0x1f, 0xdb, 0x1f, 0x59, 0xf4, 0x19, 0x90, 0xc3, 0x44, 0xf0, 0x4c, 0xa0, 0x93, - 0x63, 0x91, 0xa6, 0xfc, 0xb9, 0x58, 0x95, 0x6b, 0xa7, 0x9a, 0xeb, 0x22, 0xaf, 0x76, 0x25, 0xaf, - 0xf4, 0x0e, 0x90, 0x9e, 0x08, 0x45, 0x26, 0xf4, 0x39, 0x7f, 0x8b, 0x5d, 0x7a, 0x6e, 0x38, 0xac, - 0x5e, 0x4b, 0x6e, 0x81, 0x2b, 0x87, 0x06, 0x3a, 0x6b, 0xdd, 0x6d, 0xcb, 0x0c, 0x15, 0x93, 0x84, - 0xa1, 0x0a, 0xeb, 0x81, 0xe6, 0x46, 0xfb, 0x19, 0x52, 0x75, 0x58, 0x09, 0xd0, 0xef, 0x2c, 0xe3, - 0x0d, 0xe9, 0xff, 0xcd, 0x88, 0xe7, 0xba, 0xeb, 0x1d, 0xcd, 0xc1, 0x41, 0x0e, 0x1d, 0xc9, 0xa1, - 0x3a, 0x83, 0x96, 0xd1, 0x70, 0x17, 0x69, 0xdc, 0x37, 0xf9, 0xb9, 0x2c, 0x0b, 0xea, 0xc3, 0x0d, - 0x65, 0x61, 0xff, 0x82, 0x07, 0x21, 0x1f, 0x86, 0xff, 0xa8, 0x84, 0x73, 0x01, 0x79, 0x50, 0xc7, - 0xbd, 0xfd, 0x9e, 0x3e, 0x06, 0x46, 0xa4, 0x39, 0x94, 0x27, 0xea, 0x31, 0x1f, 0x0b, 0x6d, 0x0d, - 0xbf, 0x8b, 0x3c, 0xd8, 0x6f, 0xcd, 0xc3, 0x36, 0xd4, 0xe4, 0xf9, 0x93, 0xf3, 0xdd, 0x91, 0x2e, - 0x51, 0x58, 0x91, 0x9d, 0xf7, 0x61, 0x6d, 0xe0, 0x9f, 0x89, 0x31, 0x27, 0xff, 0x87, 0x3a, 0x32, - 0x17, 0xa9, 0x3e, 0x14, 0xcd, 0xa2, 0xe4, 0xcc, 0x68, 0xe8, 0xf7, 0x96, 0x0e, 0x76, 0x29, 0xcd, - 0x39, 0x57, 0xf6, 0x82, 0x2b, 0x72, 0x1b, 0xea, 0x9a, 0x2f, 0x4e, 0x8b, 0xd7, 0x7a, 0xca, 0x68, - 0xc9, 0x2d, 0x58, 0xc3, 0xe8, 0x52, 0xcf, 0x2d, 0x89, 0x20, 0xc2, 0xb4, 0x82, 0x1e, 0x81, 0xf3, - 0x94, 0xf5, 0xe5, 0xa0, 0x40, 0xf6, 0x86, 0x86, 0x96, 0x24, 0xb9, 0x4f, 0xe3, 0x34, 0xd3, 0xb9, - 0xc7, 0x6f, 0x89, 0x9d, 0xc4, 0x89, 0xea, 0xd3, 0x36, 0xc3, 0x6f, 0xfa, 0xa3, 0x05, 0xee, 0xe3, - 0x78, 0x24, 0xc8, 0x06, 0xd8, 0xfd, 0x9e, 0x36, 0x62, 0xf7, 0x7b, 0xe4, 0x1a, 0xda, 0xd7, 0xf9, - 0xae, 0x4b, 0xff, 0x4f, 0x59, 0x9f, 0xa1, 0xcf, 0x9b, 0xd0, 0xec, 0xa7, 0x27, 0x49, 0x30, 0xe6, - 0xc9, 0x4c, 0xdf, 0xa4, 0x25, 0x80, 0x67, 0x34, 0xe3, 0x99, 0xba, 0xdf, 0x9a, 0x4c, 0x09, 0xe4, - 0x16, 0xd4, 0x1f, 0xb2, 0x93, 0x43, 0x69, 0xb2, 0x36, 0x6f, 0xd2, 0xe0, 0xf4, 0x3e, 0x74, 0x24, - 0x13, 0x5c, 0x6f, 0x3a, 0xeb, 0x2a, 0xac, 0x49, 0xac, 0x60, 0xa6, 0xa5, 0xd2, 0x89, 0x5d, 0x71, - 0x42, 0x1f, 0x28, 0x0b, 0x47, 0x17, 0x22, 0xca, 0x2a, 0xbd, 0x89, 0x32, 0x1a, 0x68, 0x33, 0x25, - 0x90, 0x9b, 0x2a, 0x6a, 0x1d, 0x5e, 0x43, 0x72, 0x91, 0x32, 0x43, 0x94, 0xce, 0x00, 0x0c, 0x93, - 0x3c, 0x2d, 0xd6, 0x5a, 0xcb, 0xd6, 0x12, 0x6a, 0xda, 0x47, 0x1f, 0x51, 0x90, 0x7a, 0x85, 0x30, - 0xd3, 0x58, 0xef, 0x95, 0x8d, 0xa5, 0xea, 0xb9, 0x59, 0xd4, 0x5d, 0xf9, 0x28, 0xdb, 0xeb, 0x0c, - 0x5a, 0x15, 0x7c, 0x69, 0x8f, 0xdd, 0x2e, 0x9a, 0xc3, 0x2e, 0x8d, 0x21, 0xa2, 0x8d, 0x69, 0xf5, - 0x8a, 0xe1, 0x14, 0x40, 0xab, 0xb2, 0x69, 0xa9, 0xa7, 0x5d, 0xd8, 0x9c, 0x3f, 0xf0, 0xe6, 0xce, - 0x59, 0x84, 0x57, 0xb8, 0xfa, 0xc1, 0x82, 0xf6, 0x61, 0x98, 0xa7, 0x99, 0x48, 0x8a, 0x9c, 0x36, - 0x35, 0x50, 0x94, 0xb6, 0x04, 0x96, 0x57, 0x97, 0xec, 0x40, 0x4d, 0x66, 0x5c, 0x1d, 0xee, 0x6a, - 0x21, 0x14, 0x5c, 0xa9, 0x84, 0xfb, 0xa6, 0x4a, 0xd0, 0x67, 0xd0, 0x38, 0x18, 0xf4, 0x1f, 0x26, - 0x71, 0x3e, 0x59, 0x1a, 0xb1, 0x79, 0xd2, 0xd9, 0x95, 0x27, 0x5d, 0x47, 0x3d, 0x4f, 0x54, 0x54, - 0xf8, 0x22, 0xe9, 0xa8, 0x17, 0x89, 0xab, 0x11, 0x3e, 0xa5, 0x03, 0xd8, 0x52, 0xe1, 0xca, 0x89, - 0x73, 0x99, 0xb1, 0x68, 0x5e, 0x11, 0x4e, 0xf9, 0x8a, 0x90, 0x46, 0xd5, 0xd4, 0xfd, 0x37, 0x8d, - 0xfe, 0x66, 0xc3, 0x16, 0x13, 0x69, 0xf0, 0x52, 0xf4, 0xa3, 0x34, 0x4b, 0x72, 0x5f, 0x4e, 0x1c, - 0xb9, 0xff, 0xb3, 0x78, 0xa8, 0x6b, 0xe1, 0x30, 0x25, 0xbc, 0xfd, 0x94, 0x10, 0x0a, 0xf5, 0xea, - 0x10, 0xa8, 0x2e, 0x30, 0x0a, 0x72, 0x07, 0xea, 0x83, 0x38, 0x4f, 0xfc, 0xa2, 0xf3, 0x71, 0x72, - 0x2b, 0xff, 0x4a, 0xc1, 0xcc, 0x02, 0xf2, 0x08, 0xc8, 0x69, 0xc2, 0xa3, 0x34, 0xe4, 0x92, 0x92, - 0xd9, 0xd6, 0x28, 0x9f, 0x27, 0x15, 0xed, 0x9c, 0x85, 0x25, 0xdb, 0xc8, 0x5e, 0xf5, 0x08, 0x7b, - 0x75, 0xe4, 0xb7, 0x61, 0xf8, 0xe9, 0x73, 0x52, 0x3d, 0xe4, 0xf7, 0x16, 0x3a, 0xd4, 0x5b, 0xc3, - 0x2d, 0x5b, 0x72, 0xcb, 0x9c, 0x82, 0xcd, 0xaf, 0xa3, 0xdf, 0x5a, 0xb0, 0x5e, 0x65, 0xb3, 0x62, - 0x5c, 0x14, 0xe5, 0xb3, 0x57, 0xbf, 0x76, 0x4c, 0xf9, 0xdc, 0x65, 0x2f, 0xcb, 0x5a, 0xf5, 0x05, - 0x14, 0xc3, 0x7f, 0xdf, 0x90, 0x9c, 0x4b, 0xd1, 0xe9, 0x42, 0xeb, 0x84, 0x27, 0x59, 0x20, 0x8d, - 0xe9, 0x7b, 0xba, 0xc6, 0xaa, 0x10, 0x15, 0x70, 0xed, 0xb5, 0x26, 0x3a, 0x8c, 0xc7, 0x13, 0xd9, - 0xad, 0x97, 0x6a, 0x26, 0x39, 0xa6, 0x93, 0x24, 0x4e, 0x4c, 0x06, 0x50, 0xa0, 0x07, 0xd0, 0x38, - 0x8d, 0x27, 0x71, 0x18, 0x3f, 0x9f, 0xad, 0x18, 0x19, 0x1e, 0xd4, 0xd5, 0xd5, 0xa0, 0x46, 0x54, - 0x93, 0x19, 0x91, 0x5e, 0x91, 0xfd, 0xee, 0xf3, 0xd0, 0xcf, 0x43, 0x9e, 0x09, 0x7c, 0x1f, 0x23, - 0xf8, 0x79, 0xcc, 0x47, 0x6a, 0x2a, 0xe8, 0xa3, 0x45, 0xbf, 0xd4, 0x0d, 0xc8, 0x31, 0x9c, 0xca, - 0x15, 0xb4, 0x8f, 0x80, 0xb9, 0x82, 0x94, 0x44, 0x3e, 0x80, 0x56, 0x65, 0xb5, 0x0e, 0x6b, 0xb3, - 0xe8, 0x53, 0x05, 0xb3, 0xea, 0x1a, 0xfa, 0x8b, 0x35, 0xb7, 0xe7, 0xb5, 0x3b, 0x57, 0xbb, 0xba, - 0x50, 0x49, 0x6a, 0x30, 0x2d, 0xc9, 0xd0, 0x8f, 0xa6, 0x7e, 0x98, 0xa7, 0x52, 0xa5, 0x2f, 0xdc, - 0x02, 0x90, 0xa1, 0xcb, 0x1f, 0x9e, 0x38, 0x37, 0x8f, 0x1b, 0x23, 0xca, 0x5f, 0xa3, 0x9e, 0xe0, - 0xa3, 0x30, 0x88, 0x04, 0xf6, 0x8b, 0xc3, 0x0a, 0x99, 0xdc, 0x51, 0x33, 0xd6, 0x34, 0xfa, 0xf6, - 0x02, 0x71, 0xd4, 0xa9, 0xc9, 0x9b, 0x52, 0x02, 0x9d, 0x45, 0x15, 0xdd, 0x06, 0xa2, 0x3a, 0x60, - 0x7f, 0x18, 0x27, 0xe6, 0xb6, 0xa5, 0x87, 0x66, 0xb8, 0xc8, 0xec, 0xaf, 0xba, 0xc4, 0xcb, 0xcc, - 0xda, 0xd5, 0xcc, 0x1e, 0x74, 0x7e, 0x7d, 0xb5, 0x63, 0xfd, 0xfe, 0x6a, 0xc7, 0xfa, 0xe3, 0xd5, - 0x8e, 0xf5, 0xd3, 0x9f, 0x3b, 0xff, 0x19, 0xae, 0xe1, 0xaf, 0xfc, 0x87, 0x7f, 0x05, 0x00, 0x00, - 0xff, 0xff, 0x6d, 0xf8, 0xe6, 0x6b, 0xed, 0x0f, 0x00, 0x00, + // 1639 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x58, 0xdf, 0x6e, 0x1b, 0x45, + 0x17, 0xff, 0x76, 0xd7, 0x8e, 0xed, 0xe3, 0x38, 0x71, 0xa6, 0xf9, 0xfa, 0x6d, 0xd2, 0x7e, 0x91, + 0x33, 0xa0, 0xd6, 0x44, 0x22, 0x88, 0xf4, 0xa2, 0x08, 0x6e, 0x9a, 0xd8, 0x69, 0x31, 0x25, 0x6d, + 0x3a, 0x49, 0x73, 0x09, 0x9a, 0xd8, 0xa3, 0x64, 0x95, 0xf5, 0xae, 0xd9, 0x5d, 0xa7, 0x76, 0x2f, + 0x90, 0x40, 0x20, 0xb8, 0xe1, 0x9e, 0x2b, 0x9e, 0x81, 0x1b, 0xde, 0x81, 0x1b, 0x24, 0x1e, 0x01, + 0x95, 0x17, 0x41, 0x73, 0x66, 0x66, 0x77, 0xed, 0x3a, 0x35, 0x44, 0xdc, 0xed, 0xf9, 0x9d, 0x99, + 0xf3, 0x7f, 0xce, 0x9c, 0x59, 0xa8, 0x0d, 0x22, 0xef, 0x92, 0x27, 0x62, 0x7b, 0x10, 0x85, 0x49, + 0x48, 0xec, 0xc1, 0xe9, 0xfa, 0xe2, 0x60, 0x78, 0xea, 0x7b, 0x5d, 0x85, 0xd0, 0x47, 0x50, 0xe9, + 0x04, 0x3d, 0x31, 0x3a, 0x10, 0x09, 0x27, 0x04, 0x0a, 0x8f, 0xc5, 0x38, 0x76, 0x9d, 0x86, 0xd5, + 0x2c, 0x33, 0xfc, 0x26, 0x77, 0x60, 0xe9, 0x38, 0xe2, 0xdd, 0x8b, 0xfd, 0x91, 0x17, 0x27, 0x22, + 0xe8, 0x0a, 0xb7, 0x80, 0xdc, 0x29, 0x94, 0xfe, 0xec, 0xc0, 0xe2, 0x43, 0x4f, 0xf8, 0xbd, 0xa7, + 0x83, 0xc4, 0x0b, 0x83, 0x58, 0x0a, 0x3b, 0x1e, 0x0f, 0x84, 0x5b, 0x6e, 0x58, 0xcd, 0x0a, 0xc3, + 0x6f, 0x72, 0x1b, 0x2a, 0x2d, 0xde, 0x3d, 0x17, 0xc8, 0x70, 0x90, 0x91, 0x01, 0x29, 0xf7, 0xc8, + 0x7b, 0xa9, 0xb4, 0xd4, 0x58, 0x06, 0x90, 0x06, 0x54, 0x8f, 0xbd, 0xbe, 0x78, 0x36, 0xe4, 0x41, + 0x32, 0xec, 0xbb, 0x45, 0xdc, 0x9d, 0x87, 0xc8, 0x4d, 0x58, 0x78, 0xea, 0xf7, 0x0e, 0xbc, 0xc0, + 0xad, 0x34, 0xac, 0xa6, 0xc3, 0x34, 0x65, 0x70, 0x3e, 0x72, 0x21, 0xc3, 0xf9, 0x28, 0x75, 0xb7, + 0x3a, 0xe9, 0xee, 0x93, 0xf0, 0x28, 0xe1, 0x41, 0x8f, 0x47, 0xbd, 0x13, 0x4f, 0xbc, 0x70, 0x17, + 0x95, 0xbb, 0x93, 0xa8, 0xdc, 0xbb, 0xc7, 0x63, 0xe1, 0xd6, 0x50, 0x22, 0x7e, 0x93, 0x75, 0x28, + 0xef, 0x79, 0x49, 0x5b, 0x0c, 0x92, 0x73, 0x77, 0xa9, 0x61, 0x35, 0x0b, 0x2c, 0xa5, 0xc9, 0x2a, + 0x14, 0x8f, 0xba, 0xdc, 0x17, 0xee, 0x32, 0x6e, 0x50, 0x04, 0xa1, 0xb0, 0xf8, 0x30, 0x8c, 0x84, + 0x77, 0x16, 0x60, 0x12, 0xdc, 0x3a, 0x3a, 0x35, 0x81, 0x91, 0xff, 0x83, 0x23, 0x5d, 0x5a, 0x69, + 0x58, 0xcd, 0xea, 0x4e, 0x75, 0x7b, 0x70, 0xba, 0xdd, 0x16, 0x5d, 0xaf, 0xcf, 0x7d, 0x26, 0x71, + 0x64, 0xf3, 0x91, 0x4b, 0x66, 0xb1, 0xf9, 0x48, 0xda, 0x24, 0x43, 0xf4, 0x3c, 0xf0, 0x12, 0xf7, + 0x06, 0x4a, 0x4f, 0x69, 0x4a, 0x61, 0xa9, 0xd3, 0x1f, 0x84, 0x51, 0xc2, 0x44, 0x3c, 0x08, 0x83, + 0x58, 0x90, 0x3a, 0x38, 0xfb, 0x51, 0xe4, 0x5a, 0xb8, 0x50, 0x7e, 0xd2, 0x2f, 0xa1, 0xbe, 0xe7, + 0x87, 0xdd, 0x8b, 0x36, 0x4f, 0x38, 0x13, 0x5f, 0x0c, 0x45, 0x9c, 0x48, 0x5f, 0x94, 0xb9, 0x6a, + 0x9d, 0x22, 0x24, 0x8a, 0xf9, 0x77, 0x6d, 0x85, 0x22, 0x21, 0xe3, 0x84, 0x51, 0x54, 0xe9, 0xc2, + 0x6f, 0x8c, 0xc5, 0x39, 0x8f, 0x7a, 0x98, 0xe3, 0x02, 0x53, 0x84, 0x44, 0x51, 0x13, 0xd6, 0x45, + 0x81, 0x29, 0x82, 0x76, 0x60, 0x25, 0xa7, 0x5f, 0x9b, 0x79, 0x13, 0x16, 0x58, 0xf8, 0xa2, 0xd3, + 0x8e, 0x5d, 0xab, 0xe1, 0x34, 0x0b, 0x4c, 0x53, 0x58, 0x40, 0xa1, 0x3f, 0xec, 0x07, 0x92, 0x65, + 0x23, 0x2b, 0x03, 0xe8, 0x1a, 0x14, 0xb1, 0x9a, 0xa4, 0x97, 0xd9, 0x5e, 0xf9, 0x49, 0xbf, 0xb2, + 0xa0, 0x72, 0xc0, 0x47, 0x68, 0x48, 0x4c, 0xee, 0x43, 0xd9, 0xe4, 0x1a, 0x17, 0x55, 0x77, 0x6e, + 0xc9, 0xb8, 0xa6, 0x0b, 0xb6, 0x0d, 0x77, 0x3f, 0x48, 0xa2, 0x31, 0x4b, 0x17, 0xaf, 0x7f, 0x04, + 0xb5, 0x09, 0x96, 0xd4, 0x74, 0x21, 0xc6, 0x26, 0x9e, 0x17, 0x62, 0x2c, 0xbd, 0xbc, 0xe4, 0xfe, + 0x50, 0x60, 0x94, 0x0a, 0x4c, 0x11, 0x1f, 0xda, 0x1f, 0x58, 0xf4, 0x04, 0x48, 0x2b, 0x12, 0x3c, + 0x11, 0xa8, 0xe4, 0x40, 0xc4, 0x31, 0x3f, 0x13, 0xf3, 0x62, 0xed, 0xe4, 0x63, 0x9d, 0xc6, 0xd5, + 0xce, 0xc5, 0x95, 0x6e, 0x01, 0x69, 0x0b, 0x5f, 0x24, 0x42, 0x9f, 0xf3, 0x37, 0xc8, 0xa5, 0x17, + 0xc6, 0x86, 0xf9, 0x6b, 0xc9, 0x26, 0x14, 0x64, 0xd3, 0x40, 0x65, 0xd5, 0x9d, 0x9a, 0x8c, 0x50, + 0xda, 0x49, 0x18, 0xb2, 0x30, 0x1f, 0x28, 0xae, 0xb7, 0x9b, 0xa0, 0xa9, 0x0e, 0xcb, 0x00, 0xfa, + 0x8d, 0x65, 0xb4, 0xa1, 0xf9, 0x7f, 0xd3, 0xe3, 0x89, 0xea, 0x7a, 0x5b, 0xdb, 0xe0, 0xa0, 0x0d, + 0x75, 0x69, 0x43, 0xbe, 0x07, 0xcd, 0x32, 0xa3, 0x30, 0x6d, 0xc6, 0x03, 0x13, 0x9f, 0xeb, 0x5a, + 0x41, 0xbb, 0x70, 0x4b, 0x49, 0xd8, 0xbd, 0xe4, 0x9e, 0xcf, 0x4f, 0xfd, 0x7f, 0x94, 0xc2, 0x09, + 0x87, 0x5c, 0x28, 0xe1, 0xde, 0x4e, 0x5b, 0x1f, 0x03, 0x43, 0xd2, 0x21, 0x64, 0x27, 0xea, 0x09, + 0xef, 0x0b, 0x2d, 0x0d, 0xbf, 0xd3, 0x38, 0xd8, 0x6f, 0x8c, 0xc3, 0x2a, 0x14, 0xe5, 0xf9, 0x93, + 0xfd, 0xdd, 0x91, 0x2a, 0x91, 0x98, 0x13, 0x9d, 0x77, 0x61, 0xe1, 0xa8, 0x7b, 0x2e, 0xfa, 0x9c, + 0xbc, 0x05, 0x25, 0xb4, 0x5c, 0xc4, 0xfa, 0x50, 0x54, 0xd2, 0x94, 0x33, 0xc3, 0xa1, 0xdf, 0x5a, + 0xda, 0xd9, 0x99, 0x66, 0x4e, 0xa8, 0xb2, 0xa7, 0x54, 0x91, 0xbb, 0x50, 0xd2, 0xf6, 0x62, 0xb7, + 0x78, 0xad, 0xa6, 0x0c, 0x97, 0x6c, 0xc2, 0x02, 0x7a, 0x17, 0xbb, 0x85, 0xcc, 0x10, 0x44, 0x98, + 0x66, 0xd0, 0x7d, 0x70, 0x9e, 0xb3, 0x8e, 0x6c, 0x14, 0x68, 0xbd, 0x31, 0x43, 0x53, 0xd2, 0xb8, + 0x8f, 0xc3, 0x38, 0xd1, 0xb1, 0xc7, 0x6f, 0x89, 0x1d, 0x86, 0x91, 0xaa, 0xd3, 0x1a, 0xc3, 0x6f, + 0xfa, 0xbd, 0x05, 0x85, 0x27, 0x61, 0x4f, 0x90, 0x25, 0xb0, 0x3b, 0x6d, 0x2d, 0xc4, 0xee, 0xb4, + 0xc9, 0x1a, 0xca, 0xd7, 0xf1, 0x2e, 0x49, 0xfd, 0xcf, 0x59, 0x87, 0xa1, 0xce, 0xdb, 0x50, 0xe9, + 0xc4, 0x87, 0x91, 0xd7, 0xe7, 0xd1, 0x58, 0xdf, 0xa4, 0x19, 0x80, 0x67, 0x34, 0xe1, 0x89, 0xba, + 0xdf, 0x2a, 0x4c, 0x11, 0x64, 0x13, 0x4a, 0x8f, 0xd8, 0x61, 0x4b, 0x8a, 0x2c, 0x4e, 0x8a, 0x34, + 0x38, 0x7d, 0x00, 0x75, 0x69, 0x09, 0xae, 0x37, 0x95, 0x75, 0x13, 0x16, 0x24, 0x96, 0x5a, 0xa6, + 0xa9, 0x4c, 0x89, 0x9d, 0x53, 0x42, 0x1f, 0x2a, 0x09, 0xfb, 0x97, 0x22, 0x48, 0x72, 0xb5, 0x89, + 0x34, 0x0a, 0xa8, 0x31, 0x45, 0x90, 0xdb, 0xca, 0x6b, 0xed, 0x5e, 0x59, 0xda, 0x22, 0x69, 0x86, + 0x28, 0x1d, 0x03, 0x18, 0x4b, 0x86, 0x71, 0xba, 0xd6, 0x9a, 0xb5, 0x96, 0x50, 0x53, 0x3e, 0xfa, + 0x88, 0x82, 0xe4, 0x2b, 0x84, 0x99, 0xc2, 0x7a, 0x27, 0x2b, 0x2c, 0x95, 0xcf, 0xe5, 0x34, 0xef, + 0x4a, 0x47, 0x56, 0x5e, 0xe7, 0x50, 0xcd, 0xe1, 0x33, 0x6b, 0xec, 0x6e, 0x5a, 0x1c, 0x76, 0x26, + 0x0c, 0x11, 0x2d, 0x4c, 0xb3, 0xe7, 0x34, 0x27, 0x0f, 0xaa, 0xb9, 0x4d, 0x33, 0x35, 0x35, 0x61, + 0x79, 0xf2, 0xc0, 0x9b, 0x3b, 0x67, 0x1a, 0x9e, 0xa3, 0xea, 0x3b, 0x0b, 0x6a, 0x2d, 0x7f, 0x18, + 0x27, 0x22, 0x4a, 0x63, 0x5a, 0xd1, 0x40, 0x9a, 0xda, 0x0c, 0x98, 0x9d, 0x5d, 0xb2, 0x01, 0x45, + 0x19, 0x71, 0x75, 0xb8, 0xf3, 0x89, 0x50, 0x70, 0x2e, 0x13, 0x85, 0xab, 0x32, 0x41, 0x4f, 0xa0, + 0xbc, 0x77, 0xd4, 0x79, 0x14, 0x85, 0xc3, 0xc1, 0x4c, 0x8f, 0xcd, 0x48, 0x67, 0xe7, 0x46, 0xba, + 0xba, 0x1a, 0x4f, 0x94, 0x57, 0x38, 0x91, 0xd4, 0xd5, 0x44, 0x52, 0xd0, 0x08, 0x1f, 0xd1, 0x23, + 0x58, 0x51, 0xee, 0xca, 0x8e, 0x73, 0x9d, 0xb6, 0x68, 0xa6, 0x08, 0x27, 0x9b, 0x22, 0xa4, 0x50, + 0xd5, 0x75, 0xff, 0x4d, 0xa1, 0xbf, 0xd9, 0xb0, 0xc2, 0x44, 0xec, 0xbd, 0x14, 0x9d, 0x20, 0x4e, + 0xa2, 0x61, 0x57, 0x76, 0x1c, 0xb9, 0xff, 0x93, 0xf0, 0x54, 0xe7, 0xc2, 0x61, 0x8a, 0x78, 0xf3, + 0x29, 0x21, 0x14, 0x4a, 0xf9, 0x26, 0x90, 0x5f, 0x60, 0x18, 0x64, 0x0b, 0x4a, 0x47, 0xe1, 0x30, + 0xea, 0xa6, 0x95, 0x8f, 0x9d, 0x5b, 0xe9, 0x57, 0x0c, 0x66, 0x16, 0x90, 0xc7, 0x40, 0x8e, 0x23, + 0x1e, 0xc4, 0x3e, 0x97, 0x26, 0x99, 0x6d, 0xe5, 0x6c, 0x3c, 0xc9, 0x71, 0x27, 0x24, 0xcc, 0xd8, + 0x46, 0xb6, 0xf3, 0x47, 0xd8, 0x2d, 0xa1, 0x7d, 0x4b, 0xc6, 0x3e, 0x7d, 0x4e, 0xf2, 0x87, 0xfc, + 0xfe, 0x54, 0x85, 0xba, 0x0b, 0xb8, 0x65, 0x45, 0x6e, 0x99, 0x60, 0xb0, 0xc9, 0x75, 0xf4, 0x6b, + 0x0b, 0x16, 0xf3, 0xd6, 0xcc, 0x69, 0x17, 0x69, 0xfa, 0xec, 0xf9, 0xd3, 0x8e, 0x49, 0x5f, 0x61, + 0xd6, 0x64, 0x59, 0xcc, 0x4f, 0x40, 0x21, 0xfc, 0xef, 0x8a, 0xe0, 0x5c, 0xcb, 0x9c, 0x06, 0x54, + 0x0f, 0x79, 0x94, 0x78, 0x52, 0x98, 0xbe, 0xa7, 0x8b, 0x2c, 0x0f, 0x51, 0x01, 0x6b, 0xaf, 0x15, + 0x51, 0x2b, 0xec, 0x0f, 0x64, 0xb5, 0x5e, 0xab, 0x98, 0x64, 0x9b, 0x8e, 0xa2, 0x30, 0x32, 0x11, + 0x40, 0x82, 0xee, 0x41, 0xf9, 0x38, 0x1c, 0x84, 0x7e, 0x78, 0x36, 0x9e, 0xd3, 0x32, 0x5c, 0x28, + 0xa9, 0xab, 0x41, 0xb5, 0xa8, 0x0a, 0x33, 0x24, 0xbd, 0x21, 0xeb, 0xbd, 0xcb, 0xfd, 0xee, 0xd0, + 0xe7, 0x89, 0xc0, 0xf9, 0x18, 0xc1, 0x4f, 0x43, 0xde, 0x53, 0x5d, 0x41, 0x1f, 0x2d, 0xfa, 0xb9, + 0x2e, 0x40, 0x8e, 0xee, 0xe4, 0xae, 0xa0, 0x5d, 0x04, 0xcc, 0x15, 0xa4, 0x28, 0xf2, 0x3e, 0x54, + 0x73, 0xab, 0xb5, 0x5b, 0xcb, 0x69, 0x9d, 0x2a, 0x98, 0xe5, 0xd7, 0xd0, 0x5f, 0xac, 0x89, 0x3d, + 0xaf, 0xdd, 0xb9, 0x5a, 0xd5, 0xa5, 0x0a, 0x52, 0x99, 0x69, 0x4a, 0xba, 0xbe, 0x3f, 0xea, 0xfa, + 0xc3, 0x58, 0xb2, 0xf4, 0x85, 0x9b, 0x02, 0xd2, 0x75, 0xf9, 0xe0, 0x09, 0x87, 0x66, 0xb8, 0x31, + 0xa4, 0x7c, 0x1a, 0xb5, 0x05, 0xef, 0xf9, 0x5e, 0x20, 0xb0, 0x5e, 0x1c, 0x96, 0xd2, 0x64, 0x4b, + 0xf5, 0x58, 0x53, 0xe8, 0xab, 0x53, 0x86, 0x23, 0x4f, 0x75, 0xde, 0x98, 0x12, 0xa8, 0x4f, 0xb3, + 0xe8, 0x2a, 0x10, 0x55, 0x01, 0xbb, 0xa7, 0x61, 0x64, 0x6e, 0x5b, 0xda, 0x32, 0xcd, 0x45, 0x46, + 0x7f, 0xde, 0x25, 0x9e, 0x45, 0xd6, 0xce, 0x47, 0x96, 0x7e, 0x06, 0x4b, 0x7a, 0xb6, 0x13, 0x11, + 0x16, 0xb4, 0x0c, 0x00, 0x13, 0xdd, 0x50, 0x8e, 0x89, 0xe6, 0x55, 0x93, 0x01, 0x52, 0xce, 0x89, + 0x7c, 0x64, 0x98, 0xdb, 0x49, 0x53, 0x38, 0x1b, 0x79, 0x67, 0x81, 0xe8, 0xe1, 0x8d, 0xe1, 0x30, + 0x4d, 0xd1, 0x1f, 0x6c, 0x58, 0x55, 0x43, 0x67, 0x70, 0x26, 0xe2, 0x24, 0x53, 0x23, 0x9f, 0xd1, + 0x03, 0xec, 0xff, 0xda, 0x50, 0x45, 0xc9, 0x27, 0x73, 0xcb, 0x17, 0x3c, 0xca, 0x6c, 0x50, 0x8a, + 0xa6, 0x50, 0x79, 0x6e, 0x10, 0xd1, 0xd7, 0xb3, 0x1a, 0x42, 0xf3, 0x10, 0xd9, 0x83, 0xb2, 0x76, + 0xcd, 0x34, 0xc4, 0x3b, 0x78, 0x4b, 0xcd, 0xb0, 0xc6, 0xcc, 0xb7, 0xb1, 0x7e, 0x83, 0x19, 0x72, + 0xfd, 0x29, 0xd4, 0x26, 0x58, 0x33, 0xde, 0x60, 0xcd, 0xfc, 0x1b, 0xac, 0xba, 0x43, 0x72, 0xe3, + 0xb2, 0x96, 0x9e, 0x7f, 0x97, 0xb5, 0xe0, 0xbf, 0xb3, 0x0c, 0x88, 0xc9, 0x16, 0x38, 0xd2, 0x50, + 0x35, 0x0c, 0xbb, 0x57, 0x19, 0xca, 0xe4, 0x22, 0xfa, 0x93, 0xa5, 0x83, 0x2a, 0x34, 0xdf, 0xbc, + 0xa5, 0xef, 0xe5, 0x85, 0x6c, 0xa6, 0x42, 0xa6, 0x96, 0x6d, 0xa7, 0x8e, 0xca, 0xd5, 0xeb, 0xcf, + 0xa0, 0x3c, 0xcb, 0xbd, 0x82, 0x72, 0xef, 0xbd, 0x49, 0xf7, 0xd6, 0xae, 0xb2, 0x2c, 0xce, 0x79, + 0xb9, 0x57, 0xff, 0xf5, 0xd5, 0x86, 0xf5, 0xfb, 0xab, 0x0d, 0xeb, 0x8f, 0x57, 0x1b, 0xd6, 0x8f, + 0x7f, 0x6e, 0xfc, 0xe7, 0x74, 0x01, 0x7f, 0x10, 0xdd, 0xfb, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x38, + 0x55, 0x86, 0x89, 0x43, 0x12, 0x00, 0x00, } func (m *IndexMeta) Marshal() (dAtA []byte, err error) { @@ -4680,6 +4926,267 @@ func (m *ResizeNodeMessage) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *FieldOperation) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *FieldOperation) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *FieldOperation) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Signed) > 0 { + dAtA31 := make([]byte, len(m.Signed)*10) + var j30 int + for _, num1 := range m.Signed { + num := uint64(num1) + for num >= 1<<7 { + dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j30++ + } + dAtA31[j30] = uint8(num) + j30++ + } + i -= j30 + copy(dAtA[i:], dAtA31[:j30]) + i = encodeVarintPrivate(dAtA, i, uint64(j30)) + i-- + dAtA[i] = 0x1a + } + if len(m.Values) > 0 { + dAtA33 := make([]byte, len(m.Values)*10) + var j32 int + for _, num := range m.Values { + for num >= 1<<7 { + dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j32++ + } + dAtA33[j32] = uint8(num) + j32++ + } + i -= j32 + copy(dAtA[i:], dAtA33[:j32]) + i = encodeVarintPrivate(dAtA, i, uint64(j32)) + i-- + dAtA[i] = 0x12 + } + if len(m.RecordIDs) > 0 { + dAtA35 := make([]byte, len(m.RecordIDs)*10) + var j34 int + for _, num := range m.RecordIDs { + for num >= 1<<7 { + dAtA35[j34] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j34++ + } + dAtA35[j34] = uint8(num) + j34++ + } + i -= j34 + copy(dAtA[i:], dAtA35[:j34]) + i = encodeVarintPrivate(dAtA, i, uint64(j34)) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ShardIngestOperation) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardIngestOperation) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardIngestOperation) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.FieldOps) > 0 { + for k := range m.FieldOps { + v := m.FieldOps[k] + baseI := i + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + i -= len(k) + copy(dAtA[i:], k) + i = encodeVarintPrivate(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = encodeVarintPrivate(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0x22 + } + } + if len(m.ClearFields) > 0 { + for iNdEx := len(m.ClearFields) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.ClearFields[iNdEx]) + copy(dAtA[i:], m.ClearFields[iNdEx]) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.ClearFields[iNdEx]))) + i-- + dAtA[i] = 0x1a + } + } + if len(m.ClearRecordIDs) > 0 { + dAtA38 := make([]byte, len(m.ClearRecordIDs)*10) + var j37 int + for _, num := range m.ClearRecordIDs { + for num >= 1<<7 { + dAtA38[j37] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j37++ + } + dAtA38[j37] = uint8(num) + j37++ + } + i -= j37 + copy(dAtA[i:], dAtA38[:j37]) + i = encodeVarintPrivate(dAtA, i, uint64(j37)) + i-- + dAtA[i] = 0x12 + } + if len(m.OpType) > 0 { + i -= len(m.OpType) + copy(dAtA[i:], m.OpType) + i = encodeVarintPrivate(dAtA, i, uint64(len(m.OpType))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ShardIngestOperations) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardIngestOperations) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardIngestOperations) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Ops) > 0 { + for iNdEx := len(m.Ops) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Ops[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func (m *ShardedIngestRequest) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ShardedIngestRequest) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ShardedIngestRequest) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Ops) > 0 { + for k := range m.Ops { + v := m.Ops[k] + baseI := i + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPrivate(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + i = encodeVarintPrivate(dAtA, i, uint64(k)) + i-- + dAtA[i] = 0x8 + i = encodeVarintPrivate(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + func encodeVarintPrivate(dAtA []byte, offset int, v uint64) int { offset -= sovPrivate(v) base := offset @@ -5628,6 +6135,124 @@ func (m *ResizeNodeMessage) Size() (n int) { return n } +func (m *FieldOperation) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.RecordIDs) > 0 { + l = 0 + for _, e := range m.RecordIDs { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if len(m.Values) > 0 { + l = 0 + for _, e := range m.Values { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if len(m.Signed) > 0 { + l = 0 + for _, e := range m.Signed { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardIngestOperation) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.OpType) + if l > 0 { + n += 1 + l + sovPrivate(uint64(l)) + } + if len(m.ClearRecordIDs) > 0 { + l = 0 + for _, e := range m.ClearRecordIDs { + l += sovPrivate(uint64(e)) + } + n += 1 + sovPrivate(uint64(l)) + l + } + if len(m.ClearFields) > 0 { + for _, s := range m.ClearFields { + l = len(s) + n += 1 + l + sovPrivate(uint64(l)) + } + } + if len(m.FieldOps) > 0 { + for k, v := range m.FieldOps { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovPrivate(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovPrivate(uint64(len(k))) + l + n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardIngestOperations) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Ops) > 0 { + for _, e := range m.Ops { + l = e.Size() + n += 1 + l + sovPrivate(uint64(l)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ShardedIngestRequest) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Ops) > 0 { + for k, v := range m.Ops { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovPrivate(uint64(l)) + } + mapEntrySize := 1 + sovPrivate(uint64(k)) + l + n += mapEntrySize + 1 + sovPrivate(uint64(mapEntrySize)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovPrivate(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -5709,10 +6334,7 @@ func (m *IndexMeta) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6149,10 +6771,7 @@ func (m *FieldOptions) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6235,10 +6854,7 @@ func (m *ImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6423,10 +7039,7 @@ func (m *BlockDataRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6629,10 +7242,7 @@ func (m *BlockDataResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6759,10 +7369,7 @@ func (m *Cache) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -6909,7 +7516,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > postIndex { @@ -6926,10 +7533,7 @@ func (m *MaxShards) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7063,10 +7667,7 @@ func (m *CreateShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7149,10 +7750,7 @@ func (m *DeleteIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7290,10 +7888,7 @@ func (m *CreateIndexMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7463,10 +8058,7 @@ func (m *CreateFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7581,10 +8173,7 @@ func (m *DeleteFieldMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7718,10 +8307,7 @@ func (m *DeleteAvailableShardMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7891,10 +8477,7 @@ func (m *Field) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -7979,10 +8562,7 @@ func (m *Schema) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8154,10 +8734,7 @@ func (m *Index) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8291,10 +8868,7 @@ func (m *URI) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8501,10 +9075,7 @@ func (m *Node) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8619,10 +9190,7 @@ func (m *NodeStateMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8728,10 +9296,7 @@ func (m *NodeEventMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -8888,10 +9453,7 @@ func (m *NodeStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9027,10 +9589,7 @@ func (m *IndexStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9208,10 +9767,7 @@ func (m *FieldStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9396,10 +9952,7 @@ func (m *ClusterStatus) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9552,10 +10105,7 @@ func (m *BSIGroup) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9702,10 +10252,7 @@ func (m *CreateViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -9852,10 +10399,7 @@ func (m *DeleteViewMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10137,10 +10681,7 @@ func (m *ResizeInstruction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10342,10 +10883,7 @@ func (m *ResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10483,10 +11021,7 @@ func (m *TranslationResizeSource) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10624,10 +11159,7 @@ func (m *ResizeInstructionComplete) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10742,10 +11274,7 @@ func (m *Topology) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10796,10 +11325,7 @@ func (m *RecalculateCaches) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10850,10 +11376,7 @@ func (m *LoadSchemaMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -10972,10 +11495,7 @@ func (m *TransactionMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11172,10 +11692,7 @@ func (m *Transaction) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11226,10 +11743,7 @@ func (m *TransactionStats) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11280,10 +11794,7 @@ func (m *ResizeAbortMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPrivate - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { @@ -11398,10 +11909,857 @@ func (m *ResizeNodeMessage) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } - if (iNdEx + skippy) < 0 { + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *FieldOperation) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: FieldOperation: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: FieldOperation: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RecordIDs = append(m.RecordIDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.RecordIDs) == 0 { + m.RecordIDs = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.RecordIDs = append(m.RecordIDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field RecordIDs", wireType) + } + case 2: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Values = append(m.Values, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.Values) == 0 { + m.Values = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Values = append(m.Values, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Values", wireType) + } + case 3: + if wireType == 0 { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Signed = append(m.Signed, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.Signed) == 0 { + m.Signed = make([]int64, 0, elementCount) + } + for iNdEx < postIndex { + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.Signed = append(m.Signed, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field Signed", wireType) + } + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardIngestOperation) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardIngestOperation: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardIngestOperation: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field OpType", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.OpType = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType == 0 { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ClearRecordIDs = append(m.ClearRecordIDs, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if packedLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + var elementCount int + var count int + for _, integer := range dAtA[iNdEx:postIndex] { + if integer < 128 { + count++ + } + } + elementCount = count + if elementCount != 0 && len(m.ClearRecordIDs) == 0 { + m.ClearRecordIDs = make([]uint64, 0, elementCount) + } + for iNdEx < postIndex { + var v uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ClearRecordIDs = append(m.ClearRecordIDs, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ClearRecordIDs", wireType) + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClearFields", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.ClearFields = append(m.ClearFields, string(dAtA[iNdEx:postIndex])) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field FieldOps", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.FieldOps == nil { + m.FieldOps = make(map[string]*FieldOperation) + } + var mapkey string + var mapvalue *FieldOperation + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPrivate + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthPrivate + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthPrivate + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLengthPrivate + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &FieldOperation{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.FieldOps[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardIngestOperations) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardIngestOperations: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardIngestOperations: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Ops = append(m.Ops, &ShardIngestOperation{}) + if err := m.Ops[len(m.Ops)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ShardedIngestRequest) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ShardedIngestRequest: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ShardedIngestRequest: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Ops", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPrivate + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPrivate + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Ops == nil { + m.Ops = make(map[uint64]*ShardIngestOperations) + } + var mapkey uint64 + var mapvalue *ShardIngestOperations + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPrivate + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthPrivate + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLengthPrivate + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &ShardIngestOperations{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPrivate + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Ops[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPrivate(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPrivate } if (iNdEx + skippy) > l { diff --git a/pb/private.proto b/pb/private.proto index a5a43af24..9f15939c7 100644 --- a/pb/private.proto +++ b/pb/private.proto @@ -234,4 +234,25 @@ message ResizeAbortMessage { message ResizeNodeMessage { string NodeID = 1; string Action = 2; -} \ No newline at end of file +} + +message FieldOperation { + repeated uint64 RecordIDs = 1; + repeated uint64 Values = 2; + repeated int64 Signed = 3; +} + +message ShardIngestOperation { + string OpType = 1; + repeated uint64 ClearRecordIDs = 2; + repeated string ClearFields = 3; + map FieldOps = 4; +} + +message ShardIngestOperations { + repeated ShardIngestOperation Ops = 1; +} + +message ShardedIngestRequest { + map Ops = 1; +} diff --git a/pb/public.pb.go b/pb/public.pb.go index 43ebfc641..ca1b84852 100644 --- a/pb/public.pb.go +++ b/pb/public.pb.go @@ -5980,10 +5980,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6068,10 +6065,7 @@ func (m *RowMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6194,10 +6188,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6356,10 +6347,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6486,10 +6474,7 @@ func (m *IDList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6593,10 +6578,7 @@ func (m *ExtractedIDColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6713,10 +6695,7 @@ func (m *ExtractedIDMatrix) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -6799,10 +6778,7 @@ func (m *KeyList) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7016,10 +6992,7 @@ func (m *ExtractedTableValue) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7156,10 +7129,7 @@ func (m *ExtractedTableColumn) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7274,10 +7244,7 @@ func (m *ExtractedTableField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7396,10 +7363,7 @@ func (m *ExtractedTable) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7520,10 +7484,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7642,10 +7603,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7762,10 +7720,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -7835,10 +7790,7 @@ func (m *Int64) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8008,10 +7960,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8134,10 +8083,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8273,10 +8219,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8365,10 +8308,7 @@ func (m *Decimal) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8620,10 +8560,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -8740,10 +8677,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9356,10 +9290,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -9843,10 +9774,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10308,10 +10236,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10481,10 +10406,7 @@ func (m *AtomicRecord) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10567,10 +10489,7 @@ func (m *AtomicImportResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10737,10 +10656,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -10867,10 +10783,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11061,10 +10974,7 @@ func (m *TranslateIDsRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11147,10 +11057,7 @@ func (m *TranslateIDsResponse) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11267,10 +11174,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11484,10 +11388,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { @@ -11604,10 +11505,7 @@ func (m *GroupCounts) Unmarshal(dAtA []byte) error { if err != nil { return err } - if skippy < 0 { - return ErrInvalidLengthPublic - } - if (iNdEx + skippy) < 0 { + if (skippy < 0) || (iNdEx+skippy) < 0 { return ErrInvalidLengthPublic } if (iNdEx + skippy) > l { diff --git a/pg/lookerToFeaturebase.md b/pg/lookerToFeaturebase.md new file mode 100644 index 000000000..cd7c82b9b --- /dev/null +++ b/pg/lookerToFeaturebase.md @@ -0,0 +1,652 @@ +```mermaid +sequenceDiagram + +participant 213070643358480 as c0 +participant 213070643358482 as c1 +participant 213070643358484 as c2 +participant 213070643358486 as c3 +participant 213070643358488 as c4 +participant 213070643358490 as c5 +participant 213070643358492 as c6 +participant 213070643358494 as c7 +participant 213070643358496 as c8 +participant 213070643358498 as c9 +participant 213070643358500 as c10 +participant 213070643358502 as c11 +213070643358480->>server:+SSL REQUEST +server-->>213070643358480:-SSL BACKEND ANSWER: N +213070643358480->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358480:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358480:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358480:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358480:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358480:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358480:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358480:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358480:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358480:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358480:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358480:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358480:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358480:-BACKEND KEY DATA pid=1459324827, key=1506254533 +server-->>213070643358480:-READY FOR QUERY type= +213070643358480->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358480->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358480->>server:+EXECUTE name='', nb_rows=1 +213070643358480->>server:+SYNC +server-->>213070643358480:-PARSE COMPLETE +server-->>213070643358480:-BIND COMPLETE +server-->>213070643358480:-COMMAND COMPLETE command='SET' +server-->>213070643358480:-READY FOR QUERY type= +213070643358480->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358480->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358480->>server:+EXECUTE name='', nb_rows=1 +213070643358480->>server:+SYNC +server-->>213070643358480:-PARSE COMPLETE +server-->>213070643358480:-BIND COMPLETE +server-->>213070643358480:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358480:-COMMAND COMPLETE command='SET' +server-->>213070643358480:-READY FOR QUERY type= +213070643358480->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358480->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358480->>server:+DESCRIBE kind='P', name='' +213070643358480->>server:+EXECUTE name='', nb_rows=1 +213070643358480->>server:+SYNC +server-->>213070643358480:-PARSE COMPLETE +server-->>213070643358480:-BIND COMPLETE +server-->>213070643358480:-NO DATA +server-->>213070643358480:-EMPTY QUERY RESPONSE +server-->>213070643358480:-READY FOR QUERY type= +213070643358480->>server:+DISCONNECT +213070643358482->>server:+SSL REQUEST +server-->>213070643358482:-SSL BACKEND ANSWER: N +213070643358482->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358482:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358482:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358482:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358482:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358482:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358482:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358482:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358482:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358482:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358482:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358482:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358482:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358482:-BACKEND KEY DATA pid=1742342691, key=1299317425 +server-->>213070643358482:-READY FOR QUERY type= +213070643358482->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358482->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358482->>server:+EXECUTE name='', nb_rows=1 +213070643358482->>server:+SYNC +server-->>213070643358482:-PARSE COMPLETE +server-->>213070643358482:-BIND COMPLETE +server-->>213070643358482:-COMMAND COMPLETE command='SET' +server-->>213070643358482:-READY FOR QUERY type= +213070643358482->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358482->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358482->>server:+EXECUTE name='', nb_rows=1 +213070643358482->>server:+SYNC +server-->>213070643358482:-PARSE COMPLETE +server-->>213070643358482:-BIND COMPLETE +server-->>213070643358482:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358482:-COMMAND COMPLETE command='SET' +server-->>213070643358482:-READY FOR QUERY type= +213070643358482->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358482->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358482->>server:+DESCRIBE kind='P', name='' +213070643358482->>server:+EXECUTE name='', nb_rows=1 +213070643358482->>server:+SYNC +server-->>213070643358482:-PARSE COMPLETE +server-->>213070643358482:-BIND COMPLETE +server-->>213070643358482:-NO DATA +server-->>213070643358482:-EMPTY QUERY RESPONSE +server-->>213070643358482:-READY FOR QUERY type= +213070643358482->>server:+DISCONNECT +213070643358484->>server:+SSL REQUEST +server-->>213070643358484:-SSL BACKEND ANSWER: N +213070643358484->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358484:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358484:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358484:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358484:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358484:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358484:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358484:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358484:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358484:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358484:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358484:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358484:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358484:-BACKEND KEY DATA pid=700852804, key=1377869267 +server-->>213070643358484:-READY FOR QUERY type= +213070643358484->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358484->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358484->>server:+EXECUTE name='', nb_rows=1 +213070643358484->>server:+SYNC +server-->>213070643358484:-PARSE COMPLETE +server-->>213070643358484:-BIND COMPLETE +server-->>213070643358484:-COMMAND COMPLETE command='SET' +server-->>213070643358484:-READY FOR QUERY type= +213070643358484->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358484->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358484->>server:+EXECUTE name='', nb_rows=1 +213070643358484->>server:+SYNC +server-->>213070643358484:-PARSE COMPLETE +server-->>213070643358484:-BIND COMPLETE +server-->>213070643358484:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358484:-COMMAND COMPLETE command='SET' +server-->>213070643358484:-READY FOR QUERY type= +213070643358484->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358484->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358484->>server:+DESCRIBE kind='P', name='' +213070643358484->>server:+EXECUTE name='', nb_rows=1 +213070643358484->>server:+SYNC +server-->>213070643358484:-PARSE COMPLETE +server-->>213070643358484:-BIND COMPLETE +server-->>213070643358484:-NO DATA +server-->>213070643358484:-EMPTY QUERY RESPONSE +server-->>213070643358484:-READY FOR QUERY type= +213070643358484->>server:+DISCONNECT +213070643358486->>server:+SSL REQUEST +server-->>213070643358486:-SSL BACKEND ANSWER: N +213070643358486->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358486:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358486:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358486:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358486:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358486:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358486:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358486:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358486:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358486:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358486:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358486:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358486:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358486:-BACKEND KEY DATA pid=413235241, key=1302652759 +server-->>213070643358486:-READY FOR QUERY type= +213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358486->>server:+EXECUTE name='', nb_rows=1 +213070643358486->>server:+SYNC +server-->>213070643358486:-PARSE COMPLETE +server-->>213070643358486:-BIND COMPLETE +server-->>213070643358486:-COMMAND COMPLETE command='SET' +server-->>213070643358486:-READY FOR QUERY type= +213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358486->>server:+EXECUTE name='', nb_rows=1 +213070643358486->>server:+SYNC +server-->>213070643358486:-PARSE COMPLETE +server-->>213070643358486:-BIND COMPLETE +server-->>213070643358486:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358486:-COMMAND COMPLETE command='SET' +server-->>213070643358486:-READY FOR QUERY type= +213070643358486->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358486->>server:+DESCRIBE kind='P', name='' +213070643358486->>server:+EXECUTE name='', nb_rows=1 +213070643358486->>server:+SYNC +server-->>213070643358486:-PARSE COMPLETE +server-->>213070643358486:-BIND COMPLETE +server-->>213070643358486:-NO DATA +server-->>213070643358486:-EMPTY QUERY RESPONSE +server-->>213070643358486:-READY FOR QUERY type= +213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SELECT pg_backend_pid() +213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358486->>server:+DESCRIBE kind='P', name='' +213070643358486->>server:+EXECUTE name='', nb_rows=0 +213070643358486->>server:+SYNC +server-->>213070643358486:-PARSE COMPLETE +server-->>213070643358486:-BIND COMPLETE +server-->>213070643358486:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='pg_backend_pid' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643358486:-DATA ROW num_values=1 ---[Value 0001]--- length=9 value='413235241' +server-->>213070643358486:-COMMAND COMPLETE command='SELECT' +server-->>213070643358486:-READY FOR QUERY type= +213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version +213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358486->>server:+DESCRIBE kind='P', name='' +213070643358486->>server:+EXECUTE name='', nb_rows=0 +213070643358486->>server:+SYNC +server-->>213070643358486:-PARSE COMPLETE +server-->>213070643358486:-BIND COMPLETE +server-->>213070643358486:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643358486:-DATA ROW num_values=1 ---[Value 0001]--- length=27 value='PostgresSQL 13.0 (molecula)' +server-->>213070643358486:-COMMAND COMPLETE command='SELECT' +server-->>213070643358486:-READY FOR QUERY type= +213070643358486->>server:+PARSE name='', num_params=0, params_type=, query= SELECT COUNT(*) FROM pg_type AS t0, pg_aggregate AS t1, pg_settings AS t2, pg_settings AS t3 +213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358486->>server:+DESCRIBE kind='P', name='' +213070643358486->>server:+EXECUTE name='', nb_rows=0 +213070643358486->>server:+SYNC +213070643358488->>server:+SSL REQUEST +server-->>213070643358488:-SSL BACKEND ANSWER: N +213070643358488->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358488:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358488:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358488:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358488:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358488:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358488:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358488:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358488:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358488:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358488:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358488:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358488:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358488:-BACKEND KEY DATA pid=180554708, key=1504602717 +server-->>213070643358488:-READY FOR QUERY type= +213070643358488->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358488->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358488->>server:+EXECUTE name='', nb_rows=1 +213070643358488->>server:+SYNC +server-->>213070643358488:-PARSE COMPLETE +server-->>213070643358488:-BIND COMPLETE +server-->>213070643358488:-COMMAND COMPLETE command='SET' +server-->>213070643358488:-READY FOR QUERY type= +213070643358488->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358488->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358488->>server:+EXECUTE name='', nb_rows=1 +213070643358488->>server:+SYNC +server-->>213070643358488:-PARSE COMPLETE +server-->>213070643358488:-BIND COMPLETE +server-->>213070643358488:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358488:-COMMAND COMPLETE command='SET' +server-->>213070643358488:-READY FOR QUERY type= +213070643358488->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358488->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358488->>server:+DESCRIBE kind='P', name='' +213070643358488->>server:+EXECUTE name='', nb_rows=1 +213070643358488->>server:+SYNC +server-->>213070643358488:-PARSE COMPLETE +server-->>213070643358488:-BIND COMPLETE +server-->>213070643358488:-NO DATA +server-->>213070643358488:-EMPTY QUERY RESPONSE +server-->>213070643358488:-READY FOR QUERY type= +213070643358488->>server:+DISCONNECT +213070643358490->>server:+SSL REQUEST +server-->>213070643358490:-SSL BACKEND ANSWER: N +213070643358490->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358490:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358490:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358490:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358490:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358490:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358490:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358490:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358490:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358490:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358490:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358490:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358490:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358490:-BACKEND KEY DATA pid=1713101217, key=29850369 +server-->>213070643358490:-READY FOR QUERY type= +213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358490->>server:+EXECUTE name='', nb_rows=1 +213070643358490->>server:+SYNC +server-->>213070643358490:-PARSE COMPLETE +server-->>213070643358490:-BIND COMPLETE +server-->>213070643358490:-COMMAND COMPLETE command='SET' +server-->>213070643358490:-READY FOR QUERY type= +213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358490->>server:+EXECUTE name='', nb_rows=1 +213070643358490->>server:+SYNC +server-->>213070643358490:-PARSE COMPLETE +server-->>213070643358490:-BIND COMPLETE +server-->>213070643358490:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358490:-COMMAND COMPLETE command='SET' +server-->>213070643358490:-READY FOR QUERY type= +213070643358490->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358490->>server:+DESCRIBE kind='P', name='' +213070643358490->>server:+EXECUTE name='', nb_rows=1 +213070643358490->>server:+SYNC +server-->>213070643358490:-PARSE COMPLETE +server-->>213070643358490:-BIND COMPLETE +server-->>213070643358490:-NO DATA +server-->>213070643358490:-EMPTY QUERY RESPONSE +server-->>213070643358490:-READY FOR QUERY type= +213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version +213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358490->>server:+DESCRIBE kind='P', name='' +213070643358490->>server:+EXECUTE name='', nb_rows=0 +213070643358490->>server:+SYNC +server-->>213070643358490:-PARSE COMPLETE +server-->>213070643358490:-BIND COMPLETE +server-->>213070643358490:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643358490:-DATA ROW num_values=1 ---[Value 0001]--- length=27 value='PostgresSQL 13.0 (molecula)' +server-->>213070643358490:-COMMAND COMPLETE command='SELECT' +server-->>213070643358490:-READY FOR QUERY type= +213070643358490->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358490->>server:+DESCRIBE kind='P', name='' +213070643358490->>server:+EXECUTE name='', nb_rows=1 +213070643358490->>server:+SYNC +server-->>213070643358490:-PARSE COMPLETE +server-->>213070643358490:-BIND COMPLETE +server-->>213070643358490:-NO DATA +server-->>213070643358490:-EMPTY QUERY RESPONSE +server-->>213070643358490:-READY FOR QUERY type= +213070643358490->>server:+PARSE name='', num_params=0, params_type=, query= SELECT pid as id, query as stmt, EXTRACT(seconds from query_start - NOW()) as elapsed_time FROM pg_stat_activity WHERE usename='docker' +213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358490->>server:+DESCRIBE kind='P', name='' +213070643358490->>server:+EXECUTE name='', nb_rows=0 +213070643358490->>server:+SYNC +server-->>213070643358490:-PARSE COMPLETE +server-->>213070643358490:-BIND COMPLETE +server-->>213070643358490:-ROW DESCRIPTION: num_fields=3 ---[Field 01]--- name='id' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0 ---[Field 02]--- name='stmt' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0 ---[Field 03]--- name='elapsed_time' type=701 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643358490:-DATA ROW num_values=3 ---[Value 0001]--- length=9 value='413235241' ---[Value 0002]--- length=148 value=' SELECT COUNT(*). FROM pg_type AS t0,. pg_aggregate AS t1,. pg_settings AS t2,. pg_settings AS t3.' ---[Value 0003]--- length=11 value='1.311371547' +server-->>213070643358490:-DATA ROW num_values=3 ---[Value 0001]--- length=10 value='1713101217' ---[Value 0002]--- length=190 value=' SELECT pid as id,. query as stmt,. EXTRACT(seconds from query_start - NOW()) as elapsed_time. FROM pg_stat_activity. WHERE usename='docker'.' ---[Value 0003]--- length=11 value='0.000169969' +server-->>213070643358490:-COMMAND COMPLETE command='SELECT' +server-->>213070643358490:-READY FOR QUERY type= +213070643358492->>server:+SSL REQUEST +server-->>213070643358492:-SSL BACKEND ANSWER: N +213070643358492->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358492:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358492:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358492:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358492:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358492:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358492:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358492:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358492:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358492:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358492:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358492:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358492:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358492:-BACKEND KEY DATA pid=2034361563, key=1419995666 +server-->>213070643358492:-READY FOR QUERY type= +213070643358492->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358492->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358492->>server:+EXECUTE name='', nb_rows=1 +213070643358492->>server:+SYNC +server-->>213070643358492:-PARSE COMPLETE +server-->>213070643358492:-BIND COMPLETE +server-->>213070643358492:-COMMAND COMPLETE command='SET' +server-->>213070643358492:-READY FOR QUERY type= +213070643358492->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358492->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358492->>server:+EXECUTE name='', nb_rows=1 +213070643358492->>server:+SYNC +server-->>213070643358492:-PARSE COMPLETE +server-->>213070643358492:-BIND COMPLETE +server-->>213070643358492:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358492:-COMMAND COMPLETE command='SET' +server-->>213070643358492:-READY FOR QUERY type= +213070643358492->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358492->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358492->>server:+DESCRIBE kind='P', name='' +213070643358492->>server:+EXECUTE name='', nb_rows=1 +213070643358492->>server:+SYNC +server-->>213070643358492:-PARSE COMPLETE +server-->>213070643358492:-BIND COMPLETE +server-->>213070643358492:-NO DATA +server-->>213070643358492:-EMPTY QUERY RESPONSE +server-->>213070643358492:-READY FOR QUERY type= +213070643358492->>server:+DISCONNECT +213070643358494->>server:+SSL REQUEST +server-->>213070643358494:-SSL BACKEND ANSWER: N +213070643358494->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358494:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358494:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358494:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358494:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358494:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358494:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358494:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358494:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358494:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358494:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358494:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358494:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358494:-BACKEND KEY DATA pid=964429462, key=1673405115 +server-->>213070643358494:-READY FOR QUERY type= +213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358494->>server:+EXECUTE name='', nb_rows=1 +213070643358494->>server:+SYNC +server-->>213070643358494:-PARSE COMPLETE +server-->>213070643358494:-BIND COMPLETE +server-->>213070643358494:-COMMAND COMPLETE command='SET' +server-->>213070643358494:-READY FOR QUERY type= +213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358494->>server:+EXECUTE name='', nb_rows=1 +213070643358494->>server:+SYNC +server-->>213070643358494:-PARSE COMPLETE +server-->>213070643358494:-BIND COMPLETE +server-->>213070643358494:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358494:-COMMAND COMPLETE command='SET' +server-->>213070643358494:-READY FOR QUERY type= +213070643358494->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358494->>server:+DESCRIBE kind='P', name='' +213070643358494->>server:+EXECUTE name='', nb_rows=1 +213070643358494->>server:+SYNC +server-->>213070643358494:-PARSE COMPLETE +server-->>213070643358494:-BIND COMPLETE +server-->>213070643358494:-NO DATA +server-->>213070643358494:-EMPTY QUERY RESPONSE +server-->>213070643358494:-READY FOR QUERY type= +213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=select pg_terminate_backend(413235241) +213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358494->>server:+DESCRIBE kind='P', name='' +213070643358494->>server:+EXECUTE name='', nb_rows=0 +213070643358494->>server:+SYNC +server-->>213070643358494:-PARSE COMPLETE +server-->>213070643358494:-BIND COMPLETE +server-->>213070643358494:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='pg_terminate_backend' type=16 type_len=1 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643358494:-DATA ROW num_values=1 ---[Value 0001]--- length=1 value='t' +server-->>213070643358494:-COMMAND COMPLETE command='SELECT 1' +server-->>213070643358486:-PARSE COMPLETE +server-->>213070643358486:-BIND COMPLETE +server-->>213070643358486:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='count' type=20 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643358486:-ERROR RESPONSE Severity: 'FATAL' Message: 'terminating connection due to administrator command' Code: '57P01' +server-->>213070643358494:-READY FOR QUERY type= +213070643358490->>server:+DISCONNECT +213070643358494->>server:+DISCONNECT +213070643358496->>server:+SSL REQUEST +server-->>213070643358496:-SSL BACKEND ANSWER: N +213070643358496->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358496:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358496:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358496:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358496:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358496:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358496:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358496:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358496:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358496:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358496:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358496:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358496:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358496:-BACKEND KEY DATA pid=841257432, key=992978867 +server-->>213070643358496:-READY FOR QUERY type= +213070643358496->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358496->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358496->>server:+EXECUTE name='', nb_rows=1 +213070643358496->>server:+SYNC +server-->>213070643358496:-PARSE COMPLETE +server-->>213070643358496:-BIND COMPLETE +server-->>213070643358496:-COMMAND COMPLETE command='SET' +server-->>213070643358496:-READY FOR QUERY type= +213070643358496->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358496->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358496->>server:+EXECUTE name='', nb_rows=1 +213070643358496->>server:+SYNC +server-->>213070643358496:-PARSE COMPLETE +server-->>213070643358496:-BIND COMPLETE +server-->>213070643358496:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358496:-COMMAND COMPLETE command='SET' +server-->>213070643358496:-READY FOR QUERY type= +213070643358496->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358496->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358496->>server:+DESCRIBE kind='P', name='' +213070643358496->>server:+EXECUTE name='', nb_rows=1 +213070643358496->>server:+SYNC +server-->>213070643358496:-PARSE COMPLETE +server-->>213070643358496:-BIND COMPLETE +server-->>213070643358496:-NO DATA +server-->>213070643358496:-EMPTY QUERY RESPONSE +server-->>213070643358496:-READY FOR QUERY type= +213070643358496->>server:+DISCONNECT +213070643358498->>server:+SSL REQUEST +server-->>213070643358498:-SSL BACKEND ANSWER: N +213070643358498->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358498:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358498:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358498:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358498:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358498:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358498:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358498:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358498:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358498:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358498:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358498:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358498:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358498:-BACKEND KEY DATA pid=645931978, key=579078180 +server-->>213070643358498:-READY FOR QUERY type= +213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358498->>server:+EXECUTE name='', nb_rows=1 +213070643358498->>server:+SYNC +server-->>213070643358498:-PARSE COMPLETE +server-->>213070643358498:-BIND COMPLETE +server-->>213070643358498:-COMMAND COMPLETE command='SET' +server-->>213070643358498:-READY FOR QUERY type= +213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358498->>server:+EXECUTE name='', nb_rows=1 +213070643358498->>server:+SYNC +server-->>213070643358498:-PARSE COMPLETE +server-->>213070643358498:-BIND COMPLETE +server-->>213070643358498:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358498:-COMMAND COMPLETE command='SET' +server-->>213070643358498:-READY FOR QUERY type= +213070643358498->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358498->>server:+DESCRIBE kind='P', name='' +213070643358498->>server:+EXECUTE name='', nb_rows=1 +213070643358498->>server:+SYNC +server-->>213070643358498:-PARSE COMPLETE +server-->>213070643358498:-BIND COMPLETE +server-->>213070643358498:-NO DATA +server-->>213070643358498:-EMPTY QUERY RESPONSE +server-->>213070643358498:-READY FOR QUERY type= +213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=SELECT 1 +213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358498->>server:+DESCRIBE kind='P', name='' +213070643358498->>server:+EXECUTE name='', nb_rows=0 +213070643358498->>server:+SYNC +server-->>213070643358498:-PARSE COMPLETE +server-->>213070643358498:-BIND COMPLETE +server-->>213070643358498:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='?column?' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643358498:-DATA ROW num_values=1 ---[Value 0001]--- length=1 value='1' +server-->>213070643358498:-COMMAND COMPLETE command='SELECT' +server-->>213070643358498:-READY FOR QUERY type= +213070643358498->>server:+DISCONNECT +213070643358500->>server:+SSL REQUEST +server-->>213070643358500:-SSL BACKEND ANSWER: N +213070643358500->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358500:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358500:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358500:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358500:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358500:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358500:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358500:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358500:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358500:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358500:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358500:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358500:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358500:-BACKEND KEY DATA pid=1632401438, key=1341645778 +server-->>213070643358500:-READY FOR QUERY type= +213070643358500->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358500->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358500->>server:+EXECUTE name='', nb_rows=1 +213070643358500->>server:+SYNC +server-->>213070643358500:-PARSE COMPLETE +server-->>213070643358500:-BIND COMPLETE +server-->>213070643358500:-COMMAND COMPLETE command='SET' +server-->>213070643358500:-READY FOR QUERY type= +213070643358500->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358500->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358500->>server:+EXECUTE name='', nb_rows=1 +213070643358500->>server:+SYNC +server-->>213070643358500:-PARSE COMPLETE +server-->>213070643358500:-BIND COMPLETE +server-->>213070643358500:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358500:-COMMAND COMPLETE command='SET' +server-->>213070643358500:-READY FOR QUERY type= +213070643358500->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358500->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358500->>server:+DESCRIBE kind='P', name='' +213070643358500->>server:+EXECUTE name='', nb_rows=1 +213070643358500->>server:+SYNC +server-->>213070643358500:-PARSE COMPLETE +server-->>213070643358500:-BIND COMPLETE +server-->>213070643358500:-NO DATA +server-->>213070643358500:-EMPTY QUERY RESPONSE +server-->>213070643358500:-READY FOR QUERY type= +213070643358500->>server:+DISCONNECT +213070643358502->>server:+SSL REQUEST +server-->>213070643358502:-SSL BACKEND ANSWER: N +213070643358502->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643358502:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643358502:-PARAMETER STATUS name='application_name', value='' +server-->>213070643358502:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643358502:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643358502:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643358502:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643358502:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643358502:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643358502:-PARAMETER STATUS name='server_version', value='13.0.0' +server-->>213070643358502:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643358502:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643358502:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643358502:-BACKEND KEY DATA pid=232586748, key=226557416 +server-->>213070643358502:-READY FOR QUERY type= +213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358502->>server:+EXECUTE name='', nb_rows=1 +213070643358502->>server:+SYNC +server-->>213070643358502:-PARSE COMPLETE +server-->>213070643358502:-BIND COMPLETE +server-->>213070643358502:-COMMAND COMPLETE command='SET' +server-->>213070643358502:-READY FOR QUERY type= +213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358502->>server:+EXECUTE name='', nb_rows=1 +213070643358502->>server:+SYNC +server-->>213070643358502:-PARSE COMPLETE +server-->>213070643358502:-BIND COMPLETE +server-->>213070643358502:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643358502:-COMMAND COMPLETE command='SET' +server-->>213070643358502:-READY FOR QUERY type= +213070643358502->>server:+PARSE name='', num_params=0, params_type=, query= +213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358502->>server:+DESCRIBE kind='P', name='' +213070643358502->>server:+EXECUTE name='', nb_rows=1 +213070643358502->>server:+SYNC +server-->>213070643358502:-PARSE COMPLETE +server-->>213070643358502:-BIND COMPLETE +server-->>213070643358502:-NO DATA +server-->>213070643358502:-EMPTY QUERY RESPONSE +server-->>213070643358502:-READY FOR QUERY type= +213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version +213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643358502->>server:+DESCRIBE kind='P', name='' +213070643358502->>server:+EXECUTE name='', nb_rows=0 +213070643358502->>server:+SYNC +server-->>213070643358502:-PARSE COMPLETE +server-->>213070643358502:-BIND COMPLETE +server-->>213070643358502:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643358502:-DATA ROW num_values=1 ---[Value 0001]--- length=27 value='PostgresSQL 13.0 (molecula)' +server-->>213070643358502:-COMMAND COMPLETE command='SELECT' +server-->>213070643358502:-READY FOR QUERY type= +213070643358502->>server:+DISCONNECT +``` diff --git a/pg/lookerToPostgres.md b/pg/lookerToPostgres.md new file mode 100644 index 000000000..53ef945be --- /dev/null +++ b/pg/lookerToPostgres.md @@ -0,0 +1,677 @@ +```mermaid +sequenceDiagram + +participant 213070643360888 as c0 +participant 213070643360892 as c1 +participant 213070643360896 as c2 +participant 213070643360900 as c3 +participant 213070643360904 as c4 +participant 213070643360908 as c5 +participant 213070643360912 as c6 +participant 213070643360916 as c7 +participant 213070643360920 as c8 +participant 213070643360924 as c9 +participant 213070643360928 as c10 +participant 213070643360932 as c11 +213070643360888->>server:+SSL REQUEST +server-->>213070643360888:-SSL BACKEND ANSWER: N +213070643360888->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360888:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='affbc4bf') +213070643360888->>server:+PASSWORD MESSAGE password=md5de1ae46649b137ee14e14d8fd5fc6cb6 +server-->>213070643360888:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360888:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360888:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360888:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360888:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360888:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360888:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360888:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360888:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360888:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360888:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360888:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360888:-BACKEND KEY DATA pid=97, key=2120775944 +server-->>213070643360888:-READY FOR QUERY type= +213070643360888->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360888->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360888->>server:+EXECUTE name='', nb_rows=1 +213070643360888->>server:+SYNC +server-->>213070643360888:-PARSE COMPLETE +server-->>213070643360888:-BIND COMPLETE +server-->>213070643360888:-COMMAND COMPLETE command='SET' +server-->>213070643360888:-READY FOR QUERY type= +213070643360888->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360888->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360888->>server:+EXECUTE name='', nb_rows=1 +213070643360888->>server:+SYNC +server-->>213070643360888:-PARSE COMPLETE +server-->>213070643360888:-BIND COMPLETE +server-->>213070643360888:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360888:-COMMAND COMPLETE command='SET' +server-->>213070643360888:-READY FOR QUERY type= +213070643360888->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360888->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360888->>server:+DESCRIBE kind='P', name='' +213070643360888->>server:+EXECUTE name='', nb_rows=1 +213070643360888->>server:+SYNC +server-->>213070643360888:-PARSE COMPLETE +server-->>213070643360888:-BIND COMPLETE +server-->>213070643360888:-NO DATA +server-->>213070643360888:-EMPTY QUERY RESPONSE +server-->>213070643360888:-READY FOR QUERY type= +213070643360888->>server:+DISCONNECT +213070643360892->>server:+SSL REQUEST +server-->>213070643360892:-SSL BACKEND ANSWER: N +213070643360892->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360892:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='0b76ce86') +213070643360892->>server:+PASSWORD MESSAGE password=md5b82e4b6283fe5694c0199ca058378bb8 +server-->>213070643360892:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360892:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360892:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360892:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360892:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360892:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360892:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360892:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360892:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360892:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360892:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360892:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360892:-BACKEND KEY DATA pid=98, key=1329847468 +server-->>213070643360892:-READY FOR QUERY type= +213070643360892->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360892->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360892->>server:+EXECUTE name='', nb_rows=1 +213070643360892->>server:+SYNC +server-->>213070643360892:-PARSE COMPLETE +server-->>213070643360892:-BIND COMPLETE +server-->>213070643360892:-COMMAND COMPLETE command='SET' +server-->>213070643360892:-READY FOR QUERY type= +213070643360892->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360892->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360892->>server:+EXECUTE name='', nb_rows=1 +213070643360892->>server:+SYNC +server-->>213070643360892:-PARSE COMPLETE +server-->>213070643360892:-BIND COMPLETE +server-->>213070643360892:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360892:-COMMAND COMPLETE command='SET' +server-->>213070643360892:-READY FOR QUERY type= +213070643360892->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360892->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360892->>server:+DESCRIBE kind='P', name='' +213070643360892->>server:+EXECUTE name='', nb_rows=1 +213070643360892->>server:+SYNC +server-->>213070643360892:-PARSE COMPLETE +server-->>213070643360892:-BIND COMPLETE +server-->>213070643360892:-NO DATA +server-->>213070643360892:-EMPTY QUERY RESPONSE +server-->>213070643360892:-READY FOR QUERY type= +213070643360892->>server:+DISCONNECT +213070643360896->>server:+SSL REQUEST +server-->>213070643360896:-SSL BACKEND ANSWER: N +213070643360896->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360896:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='80e88231') +213070643360896->>server:+PASSWORD MESSAGE password=md59023db05ad8c94976359641cf0ada45a +server-->>213070643360896:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360896:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360896:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360896:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360896:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360896:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360896:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360896:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360896:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360896:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360896:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360896:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360896:-BACKEND KEY DATA pid=99, key=1084480878 +server-->>213070643360896:-READY FOR QUERY type= +213070643360896->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360896->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360896->>server:+EXECUTE name='', nb_rows=1 +213070643360896->>server:+SYNC +server-->>213070643360896:-PARSE COMPLETE +server-->>213070643360896:-BIND COMPLETE +server-->>213070643360896:-COMMAND COMPLETE command='SET' +server-->>213070643360896:-READY FOR QUERY type= +213070643360896->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360896->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360896->>server:+EXECUTE name='', nb_rows=1 +213070643360896->>server:+SYNC +server-->>213070643360896:-PARSE COMPLETE +server-->>213070643360896:-BIND COMPLETE +server-->>213070643360896:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360896:-COMMAND COMPLETE command='SET' +server-->>213070643360896:-READY FOR QUERY type= +213070643360896->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360896->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360896->>server:+DESCRIBE kind='P', name='' +213070643360896->>server:+EXECUTE name='', nb_rows=1 +213070643360896->>server:+SYNC +server-->>213070643360896:-PARSE COMPLETE +server-->>213070643360896:-BIND COMPLETE +server-->>213070643360896:-NO DATA +server-->>213070643360896:-EMPTY QUERY RESPONSE +server-->>213070643360896:-READY FOR QUERY type= +213070643360896->>server:+DISCONNECT +213070643360900->>server:+SSL REQUEST +server-->>213070643360900:-SSL BACKEND ANSWER: N +213070643360900->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360900:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='68a07783') +213070643360900->>server:+PASSWORD MESSAGE password=md5136c8b8ec47347f93827ec5d7023199c +server-->>213070643360900:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360900:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360900:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360900:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360900:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360900:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360900:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360900:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360900:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360900:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360900:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360900:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360900:-BACKEND KEY DATA pid=100, key=594556468 +server-->>213070643360900:-READY FOR QUERY type= +213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360900->>server:+EXECUTE name='', nb_rows=1 +213070643360900->>server:+SYNC +server-->>213070643360900:-PARSE COMPLETE +server-->>213070643360900:-BIND COMPLETE +server-->>213070643360900:-COMMAND COMPLETE command='SET' +server-->>213070643360900:-READY FOR QUERY type= +213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360900->>server:+EXECUTE name='', nb_rows=1 +213070643360900->>server:+SYNC +server-->>213070643360900:-PARSE COMPLETE +server-->>213070643360900:-BIND COMPLETE +server-->>213070643360900:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360900:-COMMAND COMPLETE command='SET' +server-->>213070643360900:-READY FOR QUERY type= +213070643360900->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360900->>server:+DESCRIBE kind='P', name='' +213070643360900->>server:+EXECUTE name='', nb_rows=1 +213070643360900->>server:+SYNC +server-->>213070643360900:-PARSE COMPLETE +server-->>213070643360900:-BIND COMPLETE +server-->>213070643360900:-NO DATA +server-->>213070643360900:-EMPTY QUERY RESPONSE +server-->>213070643360900:-READY FOR QUERY type= +213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SELECT pg_backend_pid() +213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360900->>server:+DESCRIBE kind='P', name='' +213070643360900->>server:+EXECUTE name='', nb_rows=0 +213070643360900->>server:+SYNC +server-->>213070643360900:-PARSE COMPLETE +server-->>213070643360900:-BIND COMPLETE +server-->>213070643360900:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='pg_backend_pid' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643360900:-DATA ROW num_values=1 ---[Value 0001]--- length=3 value='100' +server-->>213070643360900:-COMMAND COMPLETE command='SELECT 1' +server-->>213070643360900:-READY FOR QUERY type= +213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version +213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360900->>server:+DESCRIBE kind='P', name='' +213070643360900->>server:+EXECUTE name='', nb_rows=0 +213070643360900->>server:+SYNC +server-->>213070643360900:-PARSE COMPLETE +server-->>213070643360900:-BIND COMPLETE +server-->>213070643360900:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643360900:-DATA ROW num_values=1 ---[Value 0001]--- length=112 value='PostgreSQL 13.3 (Debian 13.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit' +server-->>213070643360900:-COMMAND COMPLETE command='SELECT 1' +server-->>213070643360900:-READY FOR QUERY type= +213070643360900->>server:+PARSE name='', num_params=0, params_type=, query= SELECT COUNT(*) FROM pg_type AS t0, pg_aggregate AS t1, pg_settings AS t2, pg_settings AS t3 +213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360900->>server:+DESCRIBE kind='P', name='' +213070643360900->>server:+EXECUTE name='', nb_rows=0 +213070643360900->>server:+SYNC +213070643360904->>server:+SSL REQUEST +server-->>213070643360904:-SSL BACKEND ANSWER: N +213070643360904->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360904:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='a03ee692') +213070643360904->>server:+PASSWORD MESSAGE password=md5cfedb52ae0cafa87a4f1066df3ba6802 +server-->>213070643360904:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360904:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360904:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360904:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360904:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360904:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360904:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360904:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360904:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360904:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360904:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360904:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360904:-BACKEND KEY DATA pid=101, key=2854304053 +server-->>213070643360904:-READY FOR QUERY type= +213070643360904->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360904->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360904->>server:+EXECUTE name='', nb_rows=1 +213070643360904->>server:+SYNC +server-->>213070643360904:-PARSE COMPLETE +server-->>213070643360904:-BIND COMPLETE +server-->>213070643360904:-COMMAND COMPLETE command='SET' +server-->>213070643360904:-READY FOR QUERY type= +213070643360904->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360904->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360904->>server:+EXECUTE name='', nb_rows=1 +213070643360904->>server:+SYNC +server-->>213070643360904:-PARSE COMPLETE +server-->>213070643360904:-BIND COMPLETE +server-->>213070643360904:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360904:-COMMAND COMPLETE command='SET' +server-->>213070643360904:-READY FOR QUERY type= +213070643360904->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360904->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360904->>server:+DESCRIBE kind='P', name='' +213070643360904->>server:+EXECUTE name='', nb_rows=1 +213070643360904->>server:+SYNC +server-->>213070643360904:-PARSE COMPLETE +server-->>213070643360904:-BIND COMPLETE +server-->>213070643360904:-NO DATA +server-->>213070643360904:-EMPTY QUERY RESPONSE +server-->>213070643360904:-READY FOR QUERY type= +213070643360904->>server:+DISCONNECT +213070643360908->>server:+SSL REQUEST +server-->>213070643360908:-SSL BACKEND ANSWER: N +213070643360908->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360908:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='be3afb9d') +213070643360908->>server:+PASSWORD MESSAGE password=md57b5cfa30f89fc7733c814addcd548c03 +server-->>213070643360908:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360908:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360908:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360908:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360908:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360908:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360908:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360908:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360908:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360908:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360908:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360908:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360908:-BACKEND KEY DATA pid=102, key=3089396074 +server-->>213070643360908:-READY FOR QUERY type= +213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360908->>server:+EXECUTE name='', nb_rows=1 +213070643360908->>server:+SYNC +server-->>213070643360908:-PARSE COMPLETE +server-->>213070643360908:-BIND COMPLETE +server-->>213070643360908:-COMMAND COMPLETE command='SET' +server-->>213070643360908:-READY FOR QUERY type= +213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360908->>server:+EXECUTE name='', nb_rows=1 +213070643360908->>server:+SYNC +server-->>213070643360908:-PARSE COMPLETE +server-->>213070643360908:-BIND COMPLETE +server-->>213070643360908:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360908:-COMMAND COMPLETE command='SET' +server-->>213070643360908:-READY FOR QUERY type= +213070643360908->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360908->>server:+DESCRIBE kind='P', name='' +213070643360908->>server:+EXECUTE name='', nb_rows=1 +213070643360908->>server:+SYNC +server-->>213070643360908:-PARSE COMPLETE +server-->>213070643360908:-BIND COMPLETE +server-->>213070643360908:-NO DATA +server-->>213070643360908:-EMPTY QUERY RESPONSE +server-->>213070643360908:-READY FOR QUERY type= +213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version +213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360908->>server:+DESCRIBE kind='P', name='' +213070643360908->>server:+EXECUTE name='', nb_rows=0 +213070643360908->>server:+SYNC +server-->>213070643360908:-PARSE COMPLETE +server-->>213070643360908:-BIND COMPLETE +server-->>213070643360908:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643360908:-DATA ROW num_values=1 ---[Value 0001]--- length=112 value='PostgreSQL 13.3 (Debian 13.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit' +server-->>213070643360908:-COMMAND COMPLETE command='SELECT 1' +server-->>213070643360908:-READY FOR QUERY type= +213070643360908->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360908->>server:+DESCRIBE kind='P', name='' +213070643360908->>server:+EXECUTE name='', nb_rows=1 +213070643360908->>server:+SYNC +server-->>213070643360908:-PARSE COMPLETE +server-->>213070643360908:-BIND COMPLETE +server-->>213070643360908:-NO DATA +server-->>213070643360908:-EMPTY QUERY RESPONSE +server-->>213070643360908:-READY FOR QUERY type= +213070643360908->>server:+PARSE name='', num_params=0, params_type=, query= SELECT pid as id, query as stmt, EXTRACT(seconds from query_start - NOW()) as elapsed_time FROM pg_stat_activity WHERE usename='docker' +213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360908->>server:+DESCRIBE kind='P', name='' +213070643360908->>server:+EXECUTE name='', nb_rows=0 +213070643360908->>server:+SYNC +server-->>213070643360908:-PARSE COMPLETE +server-->>213070643360908:-BIND COMPLETE +server-->>213070643360908:-ROW DESCRIPTION: num_fields=3 ---[Field 01]--- name='id' type=23 type_len=4 type_mod=4294967295 relid=12250 attnum=3 format=0 ---[Field 02]--- name='stmt' type=25 type_len=65535 type_mod=4294967295 relid=12250 attnum=20 format=0 ---[Field 03]--- name='elapsed_time' type=701 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643360908:-DATA ROW num_values=3 ---[Value 0001]--- length=2 value='81' ---[Value 0002]--- length=0 value='' ---[Value 0003]--- length=-1 value=NULL +server-->>213070643360908:-DATA ROW num_values=3 ---[Value 0001]--- length=3 value='100' ---[Value 0002]--- length=148 value=' SELECT COUNT(*). FROM pg_type AS t0,. pg_aggregate AS t1,. pg_settings AS t2,. pg_settings AS t3.' ---[Value 0003]--- length=9 value='-1.155449' +server-->>213070643360908:-DATA ROW num_values=3 ---[Value 0001]--- length=3 value='102' ---[Value 0002]--- length=190 value=' SELECT pid as id,. query as stmt,. EXTRACT(seconds from query_start - NOW()) as elapsed_time. FROM pg_stat_activity. WHERE usename='docker'.' ---[Value 0003]--- length=8 value='0.002903' +server-->>213070643360908:-COMMAND COMPLETE command='SELECT 3' +server-->>213070643360908:-READY FOR QUERY type= +213070643360912->>server:+SSL REQUEST +server-->>213070643360912:-SSL BACKEND ANSWER: N +213070643360912->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360912:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='14444412') +213070643360912->>server:+PASSWORD MESSAGE password=md59cef18c88b7988d7ac2f215fc1569c62 +server-->>213070643360912:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360912:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360912:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360912:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360912:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360912:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360912:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360912:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360912:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360912:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360912:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360912:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360912:-BACKEND KEY DATA pid=103, key=1911102642 +server-->>213070643360912:-READY FOR QUERY type= +213070643360912->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360912->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360912->>server:+EXECUTE name='', nb_rows=1 +213070643360912->>server:+SYNC +server-->>213070643360912:-PARSE COMPLETE +server-->>213070643360912:-BIND COMPLETE +server-->>213070643360912:-COMMAND COMPLETE command='SET' +server-->>213070643360912:-READY FOR QUERY type= +213070643360912->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360912->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360912->>server:+EXECUTE name='', nb_rows=1 +213070643360912->>server:+SYNC +server-->>213070643360912:-PARSE COMPLETE +server-->>213070643360912:-BIND COMPLETE +server-->>213070643360912:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360912:-COMMAND COMPLETE command='SET' +server-->>213070643360912:-READY FOR QUERY type= +213070643360912->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360912->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360912->>server:+DESCRIBE kind='P', name='' +213070643360912->>server:+EXECUTE name='', nb_rows=1 +213070643360912->>server:+SYNC +server-->>213070643360912:-PARSE COMPLETE +server-->>213070643360912:-BIND COMPLETE +server-->>213070643360912:-NO DATA +server-->>213070643360912:-EMPTY QUERY RESPONSE +server-->>213070643360912:-READY FOR QUERY type= +213070643360912->>server:+DISCONNECT +213070643360916->>server:+SSL REQUEST +server-->>213070643360916:-SSL BACKEND ANSWER: N +213070643360916->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360916:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='9eb7546f') +213070643360916->>server:+PASSWORD MESSAGE password=md5c7655226306ab14fa17e44dba876a348 +server-->>213070643360916:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360916:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360916:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360916:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360916:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360916:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360916:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360916:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360916:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360916:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360916:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360916:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360916:-BACKEND KEY DATA pid=104, key=927987783 +server-->>213070643360916:-READY FOR QUERY type= +213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360916->>server:+EXECUTE name='', nb_rows=1 +213070643360916->>server:+SYNC +server-->>213070643360916:-PARSE COMPLETE +server-->>213070643360916:-BIND COMPLETE +server-->>213070643360916:-COMMAND COMPLETE command='SET' +server-->>213070643360916:-READY FOR QUERY type= +213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360916->>server:+EXECUTE name='', nb_rows=1 +213070643360916->>server:+SYNC +server-->>213070643360916:-PARSE COMPLETE +server-->>213070643360916:-BIND COMPLETE +server-->>213070643360916:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360916:-COMMAND COMPLETE command='SET' +server-->>213070643360916:-READY FOR QUERY type= +213070643360916->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360916->>server:+DESCRIBE kind='P', name='' +213070643360916->>server:+EXECUTE name='', nb_rows=1 +213070643360916->>server:+SYNC +server-->>213070643360916:-PARSE COMPLETE +server-->>213070643360916:-BIND COMPLETE +server-->>213070643360916:-NO DATA +server-->>213070643360916:-EMPTY QUERY RESPONSE +server-->>213070643360916:-READY FOR QUERY type= +213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=select pg_terminate_backend(100) +213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360916->>server:+DESCRIBE kind='P', name='' +213070643360916->>server:+EXECUTE name='', nb_rows=0 +213070643360916->>server:+SYNC +server-->>213070643360916:-PARSE COMPLETE +server-->>213070643360916:-BIND COMPLETE +server-->>213070643360916:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='pg_terminate_backend' type=16 type_len=1 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643360916:-DATA ROW num_values=1 ---[Value 0001]--- length=1 value='t' +server-->>213070643360916:-COMMAND COMPLETE command='SELECT 1' +server-->>213070643360916:-READY FOR QUERY type= +server-->>213070643360900:-PARSE COMPLETE +server-->>213070643360900:-BIND COMPLETE +server-->>213070643360900:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='count' type=20 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643360900:-ERROR RESPONSE File: 'postgres.c' Severity: 'FATAL' Message: 'terminating connection due to administrator command' Code: '57P01' Routine: 'ProcessInterrupts' Line: '3090' +213070643360908->>server:+DISCONNECT +213070643360916->>server:+DISCONNECT +213070643360920->>server:+SSL REQUEST +server-->>213070643360920:-SSL BACKEND ANSWER: N +213070643360920->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360920:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='1ad4dcf7') +213070643360920->>server:+PASSWORD MESSAGE password=md554de511c5219f67a8bbe92da445fa5a2 +server-->>213070643360920:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360920:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360920:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360920:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360920:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360920:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360920:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360920:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360920:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360920:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360920:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360920:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360920:-BACKEND KEY DATA pid=105, key=3630332440 +server-->>213070643360920:-READY FOR QUERY type= +213070643360920->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360920->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360920->>server:+EXECUTE name='', nb_rows=1 +213070643360920->>server:+SYNC +server-->>213070643360920:-PARSE COMPLETE +server-->>213070643360920:-BIND COMPLETE +server-->>213070643360920:-COMMAND COMPLETE command='SET' +server-->>213070643360920:-READY FOR QUERY type= +213070643360920->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360920->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360920->>server:+EXECUTE name='', nb_rows=1 +213070643360920->>server:+SYNC +server-->>213070643360920:-PARSE COMPLETE +server-->>213070643360920:-BIND COMPLETE +server-->>213070643360920:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360920:-COMMAND COMPLETE command='SET' +server-->>213070643360920:-READY FOR QUERY type= +213070643360920->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360920->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360920->>server:+DESCRIBE kind='P', name='' +213070643360920->>server:+EXECUTE name='', nb_rows=1 +213070643360920->>server:+SYNC +server-->>213070643360920:-PARSE COMPLETE +server-->>213070643360920:-BIND COMPLETE +server-->>213070643360920:-NO DATA +server-->>213070643360920:-EMPTY QUERY RESPONSE +server-->>213070643360920:-READY FOR QUERY type= +213070643360920->>server:+DISCONNECT +213070643360924->>server:+SSL REQUEST +server-->>213070643360924:-SSL BACKEND ANSWER: N +213070643360924->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360924:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='494c352f') +213070643360924->>server:+PASSWORD MESSAGE password=md5919fc9ed056904fa0d8e1dd00625a556 +server-->>213070643360924:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360924:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360924:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360924:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360924:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360924:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360924:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360924:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360924:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360924:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360924:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360924:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360924:-BACKEND KEY DATA pid=106, key=3364308023 +server-->>213070643360924:-READY FOR QUERY type= +213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360924->>server:+EXECUTE name='', nb_rows=1 +213070643360924->>server:+SYNC +server-->>213070643360924:-PARSE COMPLETE +server-->>213070643360924:-BIND COMPLETE +server-->>213070643360924:-COMMAND COMPLETE command='SET' +server-->>213070643360924:-READY FOR QUERY type= +213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360924->>server:+EXECUTE name='', nb_rows=1 +213070643360924->>server:+SYNC +server-->>213070643360924:-PARSE COMPLETE +server-->>213070643360924:-BIND COMPLETE +server-->>213070643360924:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360924:-COMMAND COMPLETE command='SET' +server-->>213070643360924:-READY FOR QUERY type= +213070643360924->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360924->>server:+DESCRIBE kind='P', name='' +213070643360924->>server:+EXECUTE name='', nb_rows=1 +213070643360924->>server:+SYNC +server-->>213070643360924:-PARSE COMPLETE +server-->>213070643360924:-BIND COMPLETE +server-->>213070643360924:-NO DATA +server-->>213070643360924:-EMPTY QUERY RESPONSE +server-->>213070643360924:-READY FOR QUERY type= +213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=SELECT 1 +213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360924->>server:+DESCRIBE kind='P', name='' +213070643360924->>server:+EXECUTE name='', nb_rows=0 +213070643360924->>server:+SYNC +server-->>213070643360924:-PARSE COMPLETE +server-->>213070643360924:-BIND COMPLETE +server-->>213070643360924:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='?column?' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643360924:-DATA ROW num_values=1 ---[Value 0001]--- length=1 value='1' +server-->>213070643360924:-COMMAND COMPLETE command='SELECT 1' +server-->>213070643360924:-READY FOR QUERY type= +213070643360924->>server:+DISCONNECT +213070643360928->>server:+SSL REQUEST +server-->>213070643360928:-SSL BACKEND ANSWER: N +213070643360928->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360928:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='4b2173db') +213070643360928->>server:+PASSWORD MESSAGE password=md53977c39c5c7cdef7f80e74b256a8ce25 +server-->>213070643360928:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360928:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360928:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360928:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360928:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360928:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360928:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360928:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360928:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360928:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360928:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360928:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360928:-BACKEND KEY DATA pid=107, key=1988416582 +server-->>213070643360928:-READY FOR QUERY type= +213070643360928->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360928->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360928->>server:+EXECUTE name='', nb_rows=1 +213070643360928->>server:+SYNC +server-->>213070643360928:-PARSE COMPLETE +server-->>213070643360928:-BIND COMPLETE +server-->>213070643360928:-COMMAND COMPLETE command='SET' +server-->>213070643360928:-READY FOR QUERY type= +213070643360928->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360928->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360928->>server:+EXECUTE name='', nb_rows=1 +213070643360928->>server:+SYNC +server-->>213070643360928:-PARSE COMPLETE +server-->>213070643360928:-BIND COMPLETE +server-->>213070643360928:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360928:-COMMAND COMPLETE command='SET' +server-->>213070643360928:-READY FOR QUERY type= +213070643360928->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360928->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360928->>server:+DESCRIBE kind='P', name='' +213070643360928->>server:+EXECUTE name='', nb_rows=1 +213070643360928->>server:+SYNC +server-->>213070643360928:-PARSE COMPLETE +server-->>213070643360928:-BIND COMPLETE +server-->>213070643360928:-NO DATA +server-->>213070643360928:-EMPTY QUERY RESPONSE +server-->>213070643360928:-READY FOR QUERY type= +213070643360928->>server:+DISCONNECT +213070643360932->>server:+SSL REQUEST +server-->>213070643360932:-SSL BACKEND ANSWER: N +213070643360932->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO +server-->>213070643360932:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='f6d0f51d') +213070643360932->>server:+PASSWORD MESSAGE password=md5b5b322fe9bcc7c242c6b44cc8a7898e8 +server-->>213070643360932:-AUTHENTIFICATION REQUEST code=0 (SUCCESS) +server-->>213070643360932:-PARAMETER STATUS name='application_name', value='' +server-->>213070643360932:-PARAMETER STATUS name='client_encoding', value='UTF8' +server-->>213070643360932:-PARAMETER STATUS name='DateStyle', value='ISO, MDY' +server-->>213070643360932:-PARAMETER STATUS name='integer_datetimes', value='on' +server-->>213070643360932:-PARAMETER STATUS name='IntervalStyle', value='postgres' +server-->>213070643360932:-PARAMETER STATUS name='is_superuser', value='on' +server-->>213070643360932:-PARAMETER STATUS name='server_encoding', value='UTF8' +server-->>213070643360932:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)' +server-->>213070643360932:-PARAMETER STATUS name='session_authorization', value='docker' +server-->>213070643360932:-PARAMETER STATUS name='standard_conforming_strings', value='on' +server-->>213070643360932:-PARAMETER STATUS name='TimeZone', value='GMT' +server-->>213070643360932:-BACKEND KEY DATA pid=108, key=997991029 +server-->>213070643360932:-READY FOR QUERY type= +213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3 +213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360932->>server:+EXECUTE name='', nb_rows=1 +213070643360932->>server:+SYNC +server-->>213070643360932:-PARSE COMPLETE +server-->>213070643360932:-BIND COMPLETE +server-->>213070643360932:-COMMAND COMPLETE command='SET' +server-->>213070643360932:-READY FOR QUERY type= +213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver' +213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360932->>server:+EXECUTE name='', nb_rows=1 +213070643360932->>server:+SYNC +server-->>213070643360932:-PARSE COMPLETE +server-->>213070643360932:-BIND COMPLETE +server-->>213070643360932:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver' +server-->>213070643360932:-COMMAND COMPLETE command='SET' +server-->>213070643360932:-READY FOR QUERY type= +213070643360932->>server:+PARSE name='', num_params=0, params_type=, query= +213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360932->>server:+DESCRIBE kind='P', name='' +213070643360932->>server:+EXECUTE name='', nb_rows=1 +213070643360932->>server:+SYNC +server-->>213070643360932:-PARSE COMPLETE +server-->>213070643360932:-BIND COMPLETE +server-->>213070643360932:-NO DATA +server-->>213070643360932:-EMPTY QUERY RESPONSE +server-->>213070643360932:-READY FOR QUERY type= +213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version +213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params= +213070643360932->>server:+DESCRIBE kind='P', name='' +213070643360932->>server:+EXECUTE name='', nb_rows=0 +213070643360932->>server:+SYNC +server-->>213070643360932:-PARSE COMPLETE +server-->>213070643360932:-BIND COMPLETE +server-->>213070643360932:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0 +server-->>213070643360932:-DATA ROW num_values=1 ---[Value 0001]--- length=112 value='PostgreSQL 13.3 (Debian 13.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit' +server-->>213070643360932:-COMMAND COMPLETE command='SELECT 1' +server-->>213070643360932:-READY FOR QUERY type= +213070643360932->>server:+DISCONNECT +``` diff --git a/pg/message/message.go b/pg/message/message.go index caf8214d3..9fb6871ee 100644 --- a/pg/message/message.go +++ b/pg/message/message.go @@ -17,6 +17,7 @@ package message import ( "bytes" "encoding/binary" + "fmt" ) // Type is a byte indicating the type of a Postgres message. @@ -29,11 +30,11 @@ const ( // TypeReadyForQuery is a message used to indicate that the server is ready for another query. TypeReadyForQuery Type = 'Z' - // TypeCommandComplete is a message used to indicate that a query has completed. + // TypeCommandComplete is a Backend message used to indicate that a query has completed. TypeCommandComplete Type = 'C' - // TypeError is an error message. - TypeError Type = 'E' + // TypeClos is a message used to indicate that a query has completed. Frontend + TypeClose Type = 'C' // TypeRowDescription is a message indicating the column types of the result rows from a query. TypeRowDescription Type = 'T' @@ -52,6 +53,20 @@ const ( // TypeBackendKeyData contains a cancellation key for the client to use later. TypeBackendKeyData Type = 'K' + + TypeParse Type = 'P' + TypeParseComplete Type = '1' + + TypeBind Type = 'B' + TypeBindComplete Type = '2' + + TypeExecute Type = 'E' // Frontend + TypeError Type = 'E' // Backend + TypeSync Type = 'S' // Frontend + TypeParameterStatus Type = 'S' // Backend + TypeDescribe Type = 'D' // Frontend + TypeNoData Type = 'n' // backend + TypeEmptyQueryResponse Type = 'I' // backend ) // AuthenticationOK is a message indicating that authentication has completed. @@ -59,6 +74,22 @@ var AuthenticationOK = Message{ Type: TypeAuthentication, Data: []byte{0, 0, 0, 0}, } +var ParseOK = Message{ + Type: TypeParseComplete, + Data: []byte{}, +} +var BindComplete = Message{ + Type: TypeBindComplete, + Data: []byte{}, +} +var NoData = Message{ + Type: TypeNoData, + Data: []byte{}, +} +var EmptyQueryResponse = Message{ + Type: TypeEmptyQueryResponse, + Data: []byte{}, +} // Message is a Postgres message value. type Message struct { @@ -66,6 +97,49 @@ type Message struct { Data []byte } +//debug tools +func viewString(b []byte) string { + r := []rune(string(b)) + for i := range r { + if r[i] < 32 || r[i] > 126 { + r[i] = '.' + } + } + return string(r) +} +func min(a, b int) int { + if a < b { + return a + } + return b +} +func (m *Message) Dump(prefix string) { + n := len(m.Data) + rowcount := 0 + stop := (n / 8) * 8 + k := 0 + fmt.Printf("\n %s type: '%c'\n", prefix, m.Type) + for i := 0; i <= stop; i += 8 { + k++ + if i+8 < n { + rowcount = 8 + } else { + rowcount = min(k*8, n) % 8 + } + + fmt.Printf("pos %02d hex: ", i) + for j := 0; j < rowcount; j++ { + fmt.Printf("%02x ", m.Data[i+j]) + } + for j := rowcount; j < 8; j++ { + fmt.Printf(" ") + } + fmt.Printf(" '%s'\n", viewString(m.Data[i:(i+rowcount)])) + } +} + +//endTools + // TransactionStatus is the current transaction state. type TransactionStatus byte @@ -98,6 +172,14 @@ func (e *Encoder) i32(i int32) error { return err } +/* removed for linter now +func (e *Encoder) u32(i uint32) error { + binary.BigEndian.PutUint32(e.scratch[:], i) + _, err := e.buf.Write(e.scratch[:]) + return err +} +*/ + // ReadyForQuery encodes a "ready for query" message. func (e *Encoder) ReadyForQuery(status TransactionStatus) (Message, error) { e.buf.Reset() @@ -149,6 +231,9 @@ const ( // NoticeFieldHint is a suggestion of how to address the issue. NoticeFieldHint NoticeFieldType = 'H' + + // NoticeFieldHint the SQLSTATE code for the error (see Appendix A). Not localizable. Always present. + NoticeFieldCode NoticeFieldType = 'C' ) // NoticeField is a field in an error or notice. @@ -365,3 +450,95 @@ func (e *Encoder) BackendKeyData(pid, key int32) (Message, error) { Data: e.buf.Bytes(), }, nil } + +func (e *Encoder) ParameterStatus(param, value string) (Message, error) { + e.buf.Reset() + //param + NULL + value+ NULL + _, err := e.buf.WriteString(param) + if err != nil { + return Message{}, err + } + err = e.buf.WriteByte(0) + if err != nil { + return Message{}, err + } + _, err = e.buf.WriteString(value) + if err != nil { + return Message{}, err + } + err = e.buf.WriteByte(0) + if err != nil { + return Message{}, err + } + return Message{ + Type: TypeParameterStatus, + Data: e.buf.Bytes(), + }, nil +} + +type SimpleColumn struct { + Name string + Typeid int32 + Typelen int16 +} + +func (e *Encoder) EncodeColumn(name string, typeid int32, typelen int16) (Message, error) { + return e.EncodeColumns(SimpleColumn{ + Name: name, + Typeid: typeid, + Typelen: typelen, + }) +} +func (e *Encoder) EncodeColumns(cols ...SimpleColumn) (Message, error) { + + e.buf.Reset() + err := e.i16(int16(len(cols))) // number of columns in result + if err != nil { + return Message{}, nil + } + for _, col := range cols { + + _, err = e.buf.WriteString(col.Name) // column name + if err != nil { + return Message{}, err + } + err = e.buf.WriteByte(0) //null terminate + if err != nil { + return Message{}, err + } + + err = e.i32(0) //tabel id + if err != nil { + return Message{}, err + } + + err = e.i16(0) //field id(attnum) + if err != nil { + return Message{}, err + } + + err = e.i32(col.Typeid) //type_id + if err != nil { + return Message{}, err + } + + err = e.i16(col.Typelen) //type_len + if err != nil { + return Message{}, err + } + + err = e.i32(-1) //type_mod + if err != nil { + return Message{}, err + } + + err = e.i16(0) //format 0 text 1 binary + if err != nil { + return Message{}, err + } + } + return Message{ + Type: TypeRowDescription, + Data: e.buf.Bytes(), + }, nil +} diff --git a/pg/pgtest/handler.go b/pg/pgtest/handler.go index b146359d8..845aedf19 100644 --- a/pg/pgtest/handler.go +++ b/pg/pgtest/handler.go @@ -30,6 +30,12 @@ type HandlerFunc func(context.Context, pg.QueryResultWriter, pg.Query) error func (h HandlerFunc) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error { return h(ctx, w, q) } +func (h HandlerFunc) HandleSchema(ctx context.Context, portal *pg.Portal) error { + return nil +} +func (h HandlerFunc) Version() string { + return "testv1" +} var _ pg.QueryHandler = HandlerFunc(nil) diff --git a/pg/protocol.go b/pg/protocol.go index 222a9055f..e58540e1d 100644 --- a/pg/protocol.go +++ b/pg/protocol.go @@ -25,12 +25,15 @@ import ( "io" "io/ioutil" "net" + "regexp" "strings" "sync" "time" "github.com/molecula/featurebase/v2/pg/message" + "github.com/molecula/featurebase/v2/sql" "github.com/pkg/errors" + "vitess.io/vitess/go/vt/sqlparser" ) // Protocol is a Postgres protocol version. @@ -48,6 +51,9 @@ const ( // ProtocolSupported is the main protocol version supported by this package. ProtocolSupported Protocol = ProtocolPostgres30 + + // PgServerVersion is the latest version of postgres that we claim to support. + PgServerVersion = "13.0.0" ) // Major returns the major revision of the protocol. @@ -233,6 +239,377 @@ func (s *Server) handleCancel(ctx context.Context, conn net.Conn, data []byte) e return nil } +func (s *Server) SendParameterStatus(w *message.WireWriter, param, value string, encoder *message.Encoder) error { + msg, err := encoder.ParameterStatus(param, value) + if err != nil { + return err + } + err = w.WriteMessage(msg) + if err != nil { + return errors.Wrap(err, "sending parameter status") + } + return nil +} + +type Result struct { +} +type PgType byte + +// Constants used to indicated query interception +// Only pgPassOn is allowed to be processed in Featurebase query handling +const ( + pgPassOn PgType = 'x' + pgBackendPid PgType = 'a' + pgVersion PgType = 'b' + pgCountType PgType = 'c' + pgQueryTime PgType = 'd' + pgTerminate PgType = 'e' + pgEmpty PgType = 'f' + pgSetApplication PgType = 'g' + pgSelect1 PgType = 'h' + pgSchema PgType = 'i' + pgBegin PgType = 'j' +) + +type Portal struct { + Name string + Writer *message.WireWriter + commands []message.Message + Encoder *message.Encoder + mapper *sql.Mapper + sql string + pgspecial PgType + pid int32 + queryStart time.Time + server *Server + cancelNotify <-chan struct{} +} + +func (p *Portal) Reset() { + p.Name = "" + p.sql = "" + p.pgspecial = pgPassOn + p.commands = p.commands[:0] +} + +func (p *Portal) Bind() { + p.Add(message.BindComplete) +} + +var lookPQL = regexp.MustCompile(`\[.*\].*\)\z`) + +func (p *Portal) Parse(data []byte) { + p.queryStart = time.Now() + queryStr := string(bytes.Trim(data, "\x00")) + foundPQL := lookPQL.FindStringSubmatch(queryStr) + if len(foundPQL) > 0 { + + p.sql = foundPQL[0] + p.Name = "PQL" + p.pgspecial = pgPassOn + p.Add(message.ParseOK) + return + } + if strings.Contains(queryStr, "EXTRACT") { + // had to add this hack because the vitis parser doesn't handle... + /* + SELECT pid as id, + query as stmt, + EXTRACT(seconds from query_start - NOW()) as elapsed_time + FROM pg_stat_activity + WHERE usename='docker'` + */ + p.pgspecial = pgQueryTime + p.Name = "SELECT" + p.sql = queryStr + p.Add(message.ParseOK) + return + } + + if len(queryStr) > 2 { + + query, err := p.mapper.MapSQL(queryStr) + if err != nil { + return + } + + if strings.Contains(strings.ToLower(query.SQL), "select 1") { + p.pgspecial = pgSelect1 + p.Name = "SELECT" + } else { + switch query.SQLType { + case sql.SQLTypeSet: + p.Name = "SET" + set := query.Statement.(*sqlparser.Set) + p.pgspecial = 0 + for _, item := range set.Exprs { + if item.Name.String() == "application_name" { + switch item.Expr.(type) { + case *sqlparser.SQLVal: + p.pgspecial = pgSetApplication + } + } + } + case sql.SQLTypeSelect: + p.Name = "SELECT" + p.pgspecial = pgPassOn + stmt := query.Statement.(*sqlparser.Select) + for _, item := range stmt.SelectExprs { + switch expr := item.(type) { + case *sqlparser.AliasedExpr: + switch colExpr := expr.Expr.(type) { + case *sqlparser.FuncExpr: + funcName := strings.ToLower(colExpr.Name.String()) + switch funcName { + case "pg_backend_pid": + //SELECT pg_backend_pid() + p.pgspecial = pgBackendPid + case "pg_terminate_backend": + //select pg_terminate_backend(100) + p.pgspecial = pgTerminate + case "version": + //SELECT VERSION() AS version + p.pgspecial = pgVersion + } + //need to return the pid from the cancelation object + //add row description object + //add data row for item + } + } + + } + for _, item := range stmt.From { + switch from := item.(type) { + case *sqlparser.AliasedTableExpr: + tableName := from.Expr.(sqlparser.TableName).ToViewName().Name.String() + switch tableName { + case "pg_type": + p.pgspecial = pgCountType + case "pg_stat_activity": + p.pgspecial = pgQueryTime + case "tables": + p.pgspecial = pgSchema + } + } + } + p.sql = queryStr + case sql.SQLTypeBegin: + // Ignore BEGIN + p.pgspecial = pgBegin + case sql.SQLTypeShow: + p.Name = "SHOW" + p.pgspecial = pgPassOn + p.sql = queryStr + } + } + } else { + p.pgspecial = pgEmpty + } + p.Add(message.ParseOK) +} +func (p *Portal) Describe() { + // Placeholder should we need to handle the Decribe request +} + +func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) { + queryReady = true + switch p.pgspecial { + case pgBackendPid: + rowDescription, e := p.Encoder.EncodeColumn("pg_backend_pid", int32(23), 4) + if e != nil { + err = e + return + } + p.Add(rowDescription) + pid := fmt.Sprintf("%v", p.pid) + dataRow, _ := p.Encoder.TextRow(pid) + p.Add(dataRow) + //needs data row with cancel token + case pgVersion: + rowDescription, e := p.Encoder.EncodeColumn("version", int32(25), -1) + if e != nil { + err = e + return + } + p.Add(rowDescription) + mesg := fmt.Sprintf("PostgresSQL 13.0 (molecula.%v)", p.server.QueryHandler.Version()) + dataRow, _ := p.Encoder.TextRow(mesg) + p.Add(dataRow) + case pgSelect1: + rowDescription, e := p.Encoder.EncodeColumn("?column?", int32(23), 4) + if e != nil { + err = e + return + } + p.Add(rowDescription) + dataRow, _ := p.Encoder.TextRow("1") + p.Add(dataRow) + case pgCountType: + //need to block + <-p.server.lookerChannel + rowDescription, e := p.Encoder.EncodeColumn("count", int32(20), 8) + if err != nil { + err = e + return + } + p.Add(rowDescription) + errorResponse, _ := p.Encoder.Error( + message.NoticeField{ + Type: message.NoticeFieldSeverity, + Data: "FATAL", + }, + message.NoticeField{ + Type: message.NoticeFieldMessage, + Data: "terminating connection due to administrator command", + }, + message.NoticeField{ + Type: message.NoticeFieldCode, + Data: "57P01", + }, + ) + p.Add(errorResponse) + e = p.Sync() //send and Reset + + if e != nil { + err = e + return + } + return true, queryReady, nil + case pgQueryTime: + // need to return something so that the id can be queried + //need to return SELECT pid as id, query as stmt, EXTRACT(seconds from query_start - NOW()) as elapsed_time FROM pg_stat_activity + //seems like we need a map of pids to querys + e := p.server.dumpPortalsTo(p) + if e != nil { + err = e + return + } + + case pgTerminate: + //note just have 1 lock that blocks all who try to count the activities + //TODO (twg) lock this + close(p.server.lookerChannel) //release all the other blockers and allow them to terminate + p.server.lookerChannel = make(chan struct{}) //create a new one just in case + // i think it needs to return boolean true + rowDescription, e := p.Encoder.EncodeColumn("pg_terminate_backend", int32(16), 1) + if e != nil { + err = e + return + } + p.Add(rowDescription) + dataRow, _ := p.Encoder.TextRow("t") + p.Add(dataRow) + commandComplete, e := p.Encoder.CommandComplete("SELECT 1") + if e != nil { + err = e + return + } + p.Add(commandComplete) + e = p.Sync() + if e != nil { + err = e + return + } + return false, queryReady, nil + case pgEmpty: + p.Add(message.NoData) + p.Add(message.EmptyQueryResponse) + e := p.Sync() + if e != nil { + err = e + return + } + queryReady = true + return false, queryReady, nil + case pgSetApplication: + //needs to add/send status + msg, e := p.Encoder.ParameterStatus("application_name", "PostgreSQL JDBC Driver") + if e != nil { + err = e + return + } + p.Add(msg) + case pgSchema: + parts := []message.SimpleColumn{ + { + Name: "table_schema", + Typeid: int32(19), + Typelen: 64, + }, + { + Name: "table_name", + Typeid: int32(19), + Typelen: 64, + }, + } + rowDescription, e := p.Encoder.EncodeColumns(parts...) + if e != nil { + err = e + return + } + p.Add(rowDescription) + e = p.HandleSchema() + if e != nil { + err = e + return + } + + case pgPassOn: + query := SimpleQuery(p.sql) + e := p.server.handleQuery(p, query, p.cancelNotify) + if e != nil { + err = e + return + } + return + + case pgBegin: + p.Name = "BEGIN" + } + + //maybe add in the number of items in select clause + if len(p.Name) > 0 { //only send command complete for those that have names + message, _ := p.Encoder.CommandComplete(p.Name) + p.Add(message) + } + return +} + +// handleStandard handles a connection in the standard postgres wire protocol. +func (p *Portal) Sync() error { + for _, m := range p.commands { + err := p.Writer.WriteMessage(m) + if err != nil { + return err + } + } + p.Writer.Flush() + p.Reset() + return nil +} +func (p *Portal) Add(m message.Message) { + cp := message.Message{Type: m.Type, Data: make([]byte, len(m.Data))} + copy(cp.Data, m.Data) + p.commands = append(p.commands, cp) +} + +func (p *Portal) DumpComands() { + + for _, m := range p.commands { + m.Dump("DUMPING:") + } +} +func (p *Portal) HandleSchema() error { + return p.server.QueryHandler.HandleSchema(context.Background(), p) +} + +func (p *Portal) WriteMessage(m message.Message) error { + p.Add(m) + return nil +} +func (p *Portal) Flush() error { + return nil +} // handleStandard handles a connection in the standard postgres wire protocol. // The client is responsible for closing the connection when this finishes. @@ -255,7 +632,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co if err != nil { return errors.Wrap(err, "parsing parameters") } - if user, ok := params["user"]; ok { // Log the connection. s.Logger.Debugf("new postgres connection from user %q at %v", user, conn.RemoteAddr()) @@ -265,7 +641,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co } // Set up message input and output. - // Set up a reader that will preempt the connection when the context is canceled. ir := idleReader{ conn: conn, @@ -322,8 +697,56 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co if err != nil { return errors.Wrap(err, "sending authentication confirmation") } + err = s.SendParameterStatus(w, "application_name", "", &encoder) + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + err = s.SendParameterStatus(w, "client_encoding", "UTF8", &encoder) + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + err = s.SendParameterStatus(w, "DateStyle", "ISO, MDY", &encoder) + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + err = s.SendParameterStatus(w, "integer_datetimes", "on", &encoder) + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + err = s.SendParameterStatus(w, "IntervalStyle", "postgres", &encoder) + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + err = s.SendParameterStatus(w, "is_superuser", "on", &encoder) + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + err = s.SendParameterStatus(w, "server_encoding", "UTF8", &encoder) + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + err = s.SendParameterStatus(w, "server_version", PgServerVersion, &encoder) + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + /* + -PARAMETER STATUS name='TimeZone', value='GMT' + */ + err = s.SendParameterStatus(w, "session_authorization", "docker", &encoder) //TODO(twg) figure out valid values here + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + err = s.SendParameterStatus(w, "standard_conforming_strings", "on", &encoder) //TODO(twg) figure out valid values here + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } + err = s.SendParameterStatus(w, "TimeZone", "GMT", &encoder) //TODO(twg) figure out valid values here + if err != nil { + return errors.Wrap(err, "sending parameter status server version") + } var cancelNotify <-chan struct{} + var pid int32 if s.CancellationManager != nil { notify, cancel, token, err := s.CancellationManager.Token() if err != nil { @@ -332,6 +755,7 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co defer cancel() msg, err := encoder.BackendKeyData(token.PID, token.Key) + pid = token.PID if err != nil { return errors.Wrap(err, "encoding cancellation key data") } @@ -343,11 +767,23 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co } var queryReady bool + portal := &Portal{ + Writer: w, + Encoder: &encoder, + commands: make([]message.Message, 0), + mapper: sql.NewMapper(), + pid: pid, + server: s, + cancelNotify: cancelNotify, + } + s.addPortal(portal) + defer s.removePortal(portal) + //mapper.Logger = logger for { if !queryReady { // Indicate that we are ready for a query. - // TODO: provide a valid transaction state. - msg, err := encoder.ReadyForQuery(message.TransactionStatusActive) + portal.sql = "" + msg, err := encoder.ReadyForQuery(message.TransactionStatusIdle) if err != nil { return errors.Wrap(err, "sending query ready status") } @@ -373,6 +809,7 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co // However, it seems that no clients completely follow the spec, so we shouldn't rely on anything that isn't entirely straightforward. s.Logger.Debugf("postgres client sent additional data without waiting for completion") } + queryReady = true } // Read the next packet. @@ -406,12 +843,27 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co // We are done. return w.Flush() - case message.TypeSimpleQuery: - // Execute a simple query. - + case message.TypeParse: + portal.Parse(msg.Data) + case message.TypeBind: + portal.Bind() + case message.TypeExecute: + term, qr, err := portal.Execute() + if err != nil { + return err + } + if term { + return w.Flush() + } + queryReady = qr + case message.TypeSync: + err := portal.Sync() + if err != nil { + return err + } + queryReady = false + case message.TypeSimpleQuery: queryReady = false - - // Parse the query message (a null-terminated string). query := SimpleQuery(strings.TrimSuffix(string(msg.Data), "\x00")) // Execute the query. @@ -419,7 +871,10 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co if err != nil { return err } - + case message.TypeDescribe: + portal.Describe() + case message.TypeClose: + return w.Flush() default: // The message is not supported yet. // Send an error. @@ -452,6 +907,70 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co } } } +func (s *Server) addPortal(p *Portal) { + s.mu.Lock() + defer s.mu.Unlock() + s.portals = append(s.portals, p) +} +func (s *Server) removePortal(p *Portal) { + s.mu.Lock() + defer s.mu.Unlock() + for i, portal := range s.portals { + if portal.pid == p.pid { + //remove i + s.portals = append(s.portals[:i], s.portals[i+1:]...) + return + } + + } + +} +func (s *Server) dumpPortalsTo(p *Portal) error { + s.mu.Lock() + defer s.mu.Unlock() + //need to add the descrition for the 3 fields + // <-:-ROW DESCRIPTION: num_fields=3 + //---[Field 01]--- name='id' type=23 type_len=4 type_mod=4294967295 relid=12250 attnum=3 format=0 + //---[Field 02]--- name='stmt' type=25 type_len=65535 type_mod=4294967295 relid=12250 attnum=20 format=0 - + //--[Field 03]--- name='elapsed_time' type=701 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0 + parts := []message.SimpleColumn{ + { + Name: "id", + Typeid: int32(23), + Typelen: 4, + }, + { + Name: "stmt", + Typeid: int32(25), + Typelen: -1, + }, + { + Name: "elapsed_time", + Typeid: int32(701), + Typelen: 8, + }, + } + rowDescription, err := p.Encoder.EncodeColumns(parts...) + if err != nil { + return err + } + p.Add(rowDescription) + + for _, portal := range s.portals { + dataRow, err := p.Encoder.TextRow( + fmt.Sprintf("%v", portal.pid), + portal.sql, + fmt.Sprintf("%v", time.Since(portal.queryStart).Seconds())) + if err != nil { + return err + } + //if sql == "" need to put in a null record + //need to add the dararow + //also need to figure out null types + p.Add(dataRow) + } + return nil +} // handleQuery processes a single query on a connection. func (s *Server) handleQuery(w message.Writer, query Query, cancelNotify <-chan struct{}) error { @@ -497,7 +1016,6 @@ func (s *Server) handleQuery(w message.Writer, query Query, cancelNotify <-chan te: s.TypeEngine, tag: "SELECT", } - // Dispatch the query handler. qerr := s.QueryHandler.HandleQuery(ctx, qwriter, query) if qerr != nil { diff --git a/pg/query.go b/pg/query.go index cc81d490f..ebf48e84e 100644 --- a/pg/query.go +++ b/pg/query.go @@ -60,6 +60,8 @@ type QueryResultWriter interface { type QueryHandler interface { // HandleQuery executes a query and writes the results back. HandleQuery(context.Context, QueryResultWriter, Query) error + HandleSchema(context.Context, *Portal) error + Version() string } // queryResultWriter implements QueryResultWrtiter over postgres wire protocol. diff --git a/pg/server.go b/pg/server.go index 071024385..28aa6ca24 100644 --- a/pg/server.go +++ b/pg/server.go @@ -59,6 +59,9 @@ type Server struct { // CancellationManager is the cancellation manager to use. // If this is not set, no cancellations will be applied. CancellationManager CancellationManager + lookerChannel chan struct{} + mu sync.Mutex + portals []*Portal } // ServeConn serves a single connection. @@ -79,6 +82,8 @@ func (s *Server) Serve(ctx context.Context, l net.Listener) (err error) { err = cerr } }(ctx) + // TODO (twg) added for looker + s.lookerChannel = make(chan struct{}) // Wait for the listener to be closed and all connections to shut down. var wg sync.WaitGroup diff --git a/planner.go b/planner.go index 7dd39096e..7942b32b4 100644 --- a/planner.go +++ b/planner.go @@ -42,6 +42,10 @@ func (p *Planner) PlanStatement(ctx context.Context, stmt sql2.Statement) (*Stmt } func (p *Planner) planStatement(ctx context.Context, stmt sql2.Statement) (StmtNode, error) { + if err := p.checkStatement(stmt); err != nil { + return nil, err + } + switch stmt := stmt.(type) { case *sql2.SelectStatement: return p.planSelectStatement(ctx, stmt) @@ -58,21 +62,15 @@ func (p *Planner) planSelectStatement(ctx context.Context, stmt *sql2.SelectStat } func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.SelectStatement) (_ StmtNode, err error) { - // Extract table name from source. - var source *sql2.QualifiedTableName - switch src := stmt.Source.(type) { - case *sql2.JoinClause: - return nil, fmt.Errorf("cannot use JOIN in aggregate query") - case *sql2.ParenSource: - return nil, fmt.Errorf("cannot use parenthesized source in aggregate query") - case *sql2.QualifiedTableName: - source = src - case *sql2.SelectStatement: - return nil, fmt.Errorf("cannot use sub-select in aggregate query") - default: - return nil, fmt.Errorf("unexpected source type in aggregate query: %T", source) + // Handle specific case of a two-table INNER JOIN with a COUNT(). + if _, ok := stmt.Source.(*sql2.JoinClause); ok { + return p.planAggregateCountJoin(ctx, stmt) + } + + indexName, err := statementTableName(stmt) + if err != nil { + return nil, err } - indexName := sql2.IdentName(source.Name) // Convert WHERE clause. cond, err := p.planExprPQL(ctx, stmt, stmt.WhereExpr) @@ -80,48 +78,257 @@ func (p *Planner) planAggregateSelectStatement(ctx context.Context, stmt *sql2.S return nil, err } - // TODO: Support multiple aggregate calls. - if len(stmt.Columns) > 1 { - return nil, fmt.Errorf("only one call allowed in aggregate query") + // Extract calls and grouped expressions from column list. + // TODO: Recursively traverse all expression trees. + var calls []*sql2.Call + var columns []*StmtColumn + var resultCols []string + for _, c := range stmt.Columns { + columns = append(columns, &StmtColumn{ + Name: c.Name(), + Type: sql2.ExprDataType(c.Expr), + }) + + switch expr := c.Expr.(type) { + case *sql2.Call: + calls = append(calls, expr) + resultCols = append(resultCols, "_aggregate") + case *sql2.QualifiedRef: + resultCols = append(resultCols, expr.Column.Name) + default: + return nil, fmt.Errorf("unsupported expression type in aggregate query: %T", expr) + } } - // Extract aggregate call. - col := stmt.Columns[0] - var call *sql2.Call - switch expr := col.Expr.(type) { - case *sql2.Call: - call = expr - default: - return nil, fmt.Errorf("unsupported expression in aggregate query: %T", expr) + // TODO: Support multiple calls per query. + if len(calls) > 1 { + return nil, fmt.Errorf("only one aggregate call allowed") } - callName := strings.ToUpper(sql2.IdentName(call.Name)) + // Extract column names in GROUP BY clause. + var groupByCols []string + for _, expr := range stmt.GroupByExprs { + switch expr := expr.(type) { + case *sql2.QualifiedRef: + groupByCols = append(groupByCols, expr.Column.Name) + default: + return nil, fmt.Errorf("unsupported expression type in GROUP BY clause: %T", expr) + } + } + + // Extract aggregate call and build execution node. + callName := strings.ToUpper(sql2.IdentName(calls[0].Name)) switch callName { case "COUNT": - return NewCountNode(p.executor, indexName, col.Name(), cond), nil + if len(groupByCols) == 0 { + if cond == nil { + cond = &pql.Call{Name: "All"} + } + return NewCountNode(p.executor, indexName, columns[0], &pql.Call{ + Name: "Count", + Children: []*pql.Call{cond}, + }), nil + } + + var aggregate *pql.Call + if calls[0].Distinct.IsValid() { + if len(calls[0].Args) != 1 { + return nil, fmt.Errorf("distinct count must have exactly one field specified") + } + ref, ok := calls[0].Args[0].(*sql2.QualifiedRef) + if !ok { + return nil, fmt.Errorf("distinct count argument must be a field name") + } + + aggregate = &pql.Call{ + Name: "Count", + Children: []*pql.Call{{ + Name: "Distinct", + Args: map[string]interface{}{"field": ref.Column.Name}, + }}, + } + } + + return NewGroupByNode(p.executor, indexName, resultCols, groupByCols, columns, aggregate, cond), nil + + case "SUM": + if len(calls[0].Args) != 1 { + return nil, fmt.Errorf("sum must have exactly one field specified") + } + ref, ok := calls[0].Args[0].(*sql2.QualifiedRef) + if !ok { + return nil, fmt.Errorf("sum argument must be a field name") + } + + aggregate := &pql.Call{ + Name: "Sum", + Args: map[string]interface{}{"field": ref.Column.Name}, + } + + return NewGroupByNode(p.executor, indexName, resultCols, groupByCols, columns, aggregate, cond), nil + default: - return nil, fmt.Errorf("unsupported call in aggregate query: %T", callName) + return nil, fmt.Errorf("unsupported call in aggregate query: %s", callName) } // TODO: Support HAVING } -func (p *Planner) planNonAggregateSelectStatement(ctx context.Context, stmt *sql2.SelectStatement) (_ StmtNode, err error) { - // Extract table name from source. - var source *sql2.QualifiedTableName - switch src := stmt.Source.(type) { - case *sql2.JoinClause: - return nil, fmt.Errorf("cannot use JOIN in non-aggregate query") - case *sql2.ParenSource: - return nil, fmt.Errorf("cannot use parenthesized source in non-aggregate query") - case *sql2.QualifiedTableName: - source = src - case *sql2.SelectStatement: - return nil, fmt.Errorf("cannot use sub-select in non-aggregate query") - default: - return nil, fmt.Errorf("unexpected source type in non-aggregate query: %T", source) +func (p *Planner) planAggregateCountJoin(ctx context.Context, stmt *sql2.SelectStatement) (_ StmtNode, err error) { + // Ensure we have an INNER JOIN. + join := stmt.Source.(*sql2.JoinClause) // caller checked + if !join.Operator.Inner.IsValid() { + return nil, fmt.Errorf("only inner joins are currently supported") + } + + // Determine the two tables we are joining. + tbl0, ok := join.X.(*sql2.QualifiedTableName) + if !ok { + return nil, fmt.Errorf("left side of join must be a table") + } + tbl1, ok := join.Y.(*sql2.QualifiedTableName) + if !ok { + return nil, fmt.Errorf("left side of join must be a table") + } + + // Ensure INNER JOIN has an "ON" constraint. + if join.Constraint == nil { + return nil, fmt.Errorf("joins must have an ON constraint") + } + cons, ok := join.Constraint.(*sql2.OnConstraint) + if !ok { + return nil, fmt.Errorf("joins only support an ON constraint") + } + + // Determine the joined columns. + cx, ok := cons.X.(*sql2.BinaryExpr) + if !ok { + return nil, fmt.Errorf("join must use a binary expression") + } else if cx.Op != sql2.EQ { + return nil, fmt.Errorf("join must use an equality expression") + } + + // Extract join columns & validate that they reference known tables and join on "_id". + x, ok := cx.X.(*sql2.QualifiedRef) + if !ok { + return nil, fmt.Errorf("left-hand side of join expression must be a table-qualified column") + } else if x.Table.Name != tbl0.TableName() && x.Table.Name != tbl1.TableName() { + return nil, fmt.Errorf("no such table: %q", x.Table.Name) + } + + y, ok := cx.Y.(*sql2.QualifiedRef) + if !ok { + return nil, fmt.Errorf("right-hand side of join expression must be a table-qualified column") + } else if y.Table.Name != tbl0.TableName() && y.Table.Name != tbl1.TableName() { + return nil, fmt.Errorf("no such table: %q", y.Table.Name) + } + + if x.Column.Name != "_id" && y.Column.Name != "_id" { + return nil, fmt.Errorf("must join table on _id column") + } else if x.Column.Name == "_id" && y.Column.Name == "_id" { + return nil, fmt.Errorf("cannot join _id field of two tables") + } + + // Move ID column to LHS. + if x.Column.Name != "_id" { + x, y = y, x + } + + // Move parent table to LHS. + if x.Table.Name != tbl0.TableName() { + tbl0, tbl1 = tbl1, tbl0 + } + + // Ensure column expression is a single COUNT. + if len(stmt.Columns) != 1 { + return nil, fmt.Errorf("only COUNT() is supported on joined tables") + } + expr, ok := stmt.Columns[0].Expr.(*sql2.Call) + if !ok || strings.ToUpper(expr.Name.Name) != "COUNT" { + return nil, fmt.Errorf("only COUNT() is supported on joined tables") + } + + // Extract WHERE clause and separate by parent/child tables. + var cond0, cond1 sql2.Expr + for _, cond := range sql2.SplitExprTree(stmt.WhereExpr) { + tblName, ok := sql2.ExprTableName(cond) + if !ok { + return nil, fmt.Errorf("cannot filter across multiple tables in an expression") + } else if tblName == "" { + return nil, fmt.Errorf("expression must reference a table name") + } else if tblName != tbl0.TableName() && tblName != tbl1.TableName() { + return nil, fmt.Errorf("no such table: %q", tblName) + } + + // Match to parent table. + if tblName == tbl0.TableName() { + if cond0 == nil { + cond0 = cond + } else { + cond0 = &sql2.BinaryExpr{X: cond0, Op: sql2.AND, Y: cond} + } + continue + } + + // Match to child table. + if cond1 == nil { + cond1 = cond + } + cond1 = &sql2.BinaryExpr{X: cond1, Op: sql2.AND, Y: cond} + } + + // Convert conditions to PQL. + pqlCond0, err := p.planExprPQL(ctx, stmt, cond0) + if err != nil { + return nil, err + } else if pqlCond0 == nil { + pqlCond0 = &pql.Call{Name: "All"} + } + + pqlCond1, err := p.planExprPQL(ctx, stmt, cond1) + if err != nil { + return nil, err + } else if pqlCond1 == nil { + pqlCond1 = &pql.Call{ + Name: "Row", + Args: map[string]interface{}{y.Column.Name: &pql.Condition{ + Op: pql.NEQ, + }}, + } + } + + return NewCountNode(p.executor, tbl0.Name.Name, + &StmtColumn{ + Name: stmt.Columns[0].Name(), + Type: sql2.DataTypeInt, + }, + &pql.Call{ + Name: "Count", + Children: []*pql.Call{{ + Name: "Intersect", + Children: []*pql.Call{ + pqlCond0, + { + Name: "Distinct", + Children: []*pql.Call{ + pqlCond1, + }, + Args: map[string]interface{}{ + "index": tbl1.Name.Name, + "field": y.Column.Name, + }, + }, + }, + }}, + }, + ), nil +} + +func (p *Planner) planNonAggregateSelectStatement(ctx context.Context, stmt *sql2.SelectStatement) (_ StmtNode, err error) { + indexName, err := statementTableName(stmt) + if err != nil { + return nil, err } - indexName := sql2.IdentName(source.Name) // Lookup index. idx := p.executor.Holder.Index(indexName) @@ -136,57 +343,24 @@ func (p *Planner) planNonAggregateSelectStatement(ctx context.Context, stmt *sql } // Build column list. - var columnNames, columnAliases []string + var srcs []string + var columns []*StmtColumn for _, col := range stmt.Columns { - // Unqualified wildcard. - if col.Star.IsValid() { - columnNames = append(columnNames, "_id") - columnAliases = append(columnAliases, "_id") - - for _, field := range idx.Fields() { - if field.Name() == "_exists" { - continue - } - columnNames = append(columnNames, field.Name()) - columnAliases = append(columnAliases, field.Name()) - } - continue - } - // Handle expressions and qualified references. switch expr := col.Expr.(type) { - case *sql2.Ident: - columnNames = append(columnNames, expr.Name) - columnAliases = append(columnAliases, col.Name()) - case *sql2.QualifiedRef: - if tbl := sql2.IdentName(expr.Table); tbl != "" && tbl != source.TableName() { - return nil, fmt.Errorf("no such table: %q", tbl) - } - - if expr.Star.IsValid() { - columnNames = append(columnNames, "_id") - columnAliases = append(columnAliases, "_id") - - for _, field := range idx.Fields() { - if field.Name() == "_exists" { - continue - } - columnNames = append(columnNames, field.Name()) - columnAliases = append(columnAliases, field.Name()) - } - - } else { - columnNames = append(columnNames, sql2.IdentName(expr.Column)) - columnAliases = append(columnAliases, sql2.IdentName(expr.Column)) - } + srcs = append(srcs, sql2.IdentName(expr.Column)) + columns = append(columns, &StmtColumn{ + Name: sql2.IdentName(expr.Column), + Type: sql2.ExprDataType(col.Expr), + }) default: return nil, fmt.Errorf("unsupported column expression: %T", expr) } } - return NewExtractNode(p.executor, indexName, columnNames, columnAliases, cond), nil + return NewExtractNode(p.executor, indexName, srcs, columns, cond), nil } // planExprPQL returns a PQL call tree for a given expression. @@ -262,8 +436,8 @@ func (p *Planner) planBinaryExprPQL(ctx context.Context, stmt *sql2.SelectStatem case sql2.EQ, sql2.NE, sql2.LT, sql2.LE, sql2.GT, sql2.GE: // Ensure field reference exists in binary expression. x, y := expr.X, expr.Y - xIdent, xOk := x.(*sql2.Ident) - yIdent, yOk := y.(*sql2.Ident) + xRef, xOk := x.(*sql2.QualifiedRef) + yRef, yOk := y.(*sql2.QualifiedRef) if xOk && yOk { return nil, fmt.Errorf("cannot compare fields in a WHERE clause") } else if !xOk && !yOk { @@ -272,7 +446,7 @@ func (p *Planner) planBinaryExprPQL(ctx context.Context, stmt *sql2.SelectStatem // Rewrite expression so field ref is LHS. if !xOk && yOk { - xIdent, y = yIdent, x + xRef, y = yRef, x switch op { case sql2.LT: op = sql2.GT @@ -295,7 +469,7 @@ func (p *Planner) planBinaryExprPQL(ctx context.Context, stmt *sql2.SelectStatem return &pql.Call{ Name: "Row", Args: map[string]interface{}{ - sql2.IdentName(xIdent): pqlValue, + sql2.IdentName(xRef.Column): pqlValue, }, }, nil } @@ -307,7 +481,7 @@ func (p *Planner) planBinaryExprPQL(ctx context.Context, stmt *sql2.SelectStatem return &pql.Call{ Name: "Row", Args: map[string]interface{}{ - sql2.IdentName(xIdent): &pql.Condition{ + sql2.IdentName(xRef.Column): &pql.Condition{ Op: pqlOp, Value: pqlValue, }, @@ -366,6 +540,232 @@ func sqlToPQLValue(expr sql2.Expr) (interface{}, error) { } } +func (p *Planner) checkStatement(stmt sql2.Statement) error { + switch stmt := stmt.(type) { + case *sql2.SelectStatement: + return p.checkSelectStatement(stmt) + default: + return nil + } +} + +func (p *Planner) checkSelectStatement(stmt *sql2.SelectStatement) error { + if err := p.expandSelectStatementWildcards(stmt); err != nil { + return err + } + + // Type check expressions in statement. + for _, col := range stmt.Columns { + if err := p.checkExpr(&col.Expr, stmt); err != nil { + return err + } + } + + if err := p.checkExpr(&stmt.WhereExpr, stmt); err != nil { + return err + } + + for i := range stmt.GroupByExprs { + if err := p.checkExpr(&stmt.GroupByExprs[i], stmt); err != nil { + return err + } + } + + if err := p.checkExpr(&stmt.HavingExpr, stmt); err != nil { + return err + } + + for _, term := range stmt.OrderingTerms { + if err := p.checkExpr(&term.X, stmt); err != nil { + return err + } + } + + if err := p.checkExpr(&stmt.LimitExpr, stmt); err != nil { + return err + } + + if err := p.checkExpr(&stmt.OffsetExpr, stmt); err != nil { + return err + } + + return nil +} + +func (p *Planner) expandSelectStatementWildcards(stmt *sql2.SelectStatement) error { + if !stmt.HasWildcard() { + return nil + } + + indexName, err := statementTableName(stmt) + if err != nil { + return err + } + + // Look up index. + idx := p.executor.Holder.Index(indexName) + if idx == nil { + return newNotFoundError(ErrIndexNotFound, indexName) + } + + // Replace wildcards with column references. + columns := make([]*sql2.ResultColumn, 0, len(stmt.Columns)) + for _, col := range stmt.Columns { + // Unqualified wildcard. + isWildcard := col.Star.IsValid() + if ref, ok := col.Expr.(*sql2.QualifiedRef); ok && ref.Star.IsValid() { + if ref.Table.Name != indexName { + return fmt.Errorf("no such table: %q", ref.Table.Name) + } + isWildcard = true + } + + // Simply add column as-is if it is not a wildcard. + if !isWildcard { + columns = append(columns, col) + continue + } + + // Add identifier field first. + columns = append(columns, &sql2.ResultColumn{ + Expr: &sql2.QualifiedRef{ + Table: &sql2.Ident{Name: idx.Name()}, + Column: &sql2.Ident{Name: "_id"}, + }, + }) + + // Then add all fields besides the existence bit. + for _, field := range idx.Fields() { + if field.Name() == "_exists" { + continue + } + columns = append(columns, &sql2.ResultColumn{ + Expr: &sql2.QualifiedRef{ + Table: &sql2.Ident{Name: idx.Name()}, + Column: &sql2.Ident{Name: field.Name()}, + }, + }) + } + } + stmt.Columns = columns + + return nil +} +func (p *Planner) checkExpr(expr *sql2.Expr, stmt sql2.Statement) error { + if e, err := sql2.Walk(&sqlExprTypeChecker{ + holder: p.executor.Holder, + stmt: stmt, + }, *expr); err != nil { + return err + } else if e != nil { + *expr = e.(sql2.Expr) + } else { + *expr = nil + } + return nil +} + +// sqlExprTypeChecker recursively performs type checking within an expression. +// Called by sqlTypeChecker. Implements sql2.Visitor. +type sqlExprTypeChecker struct { + holder *Holder + stmt sql2.Statement // scope +} + +var _ sql2.Visitor = (*sqlExprTypeChecker)(nil) + +func (v *sqlExprTypeChecker) Visit(node sql2.Node) (_ sql2.Visitor, _ sql2.Node, err error) { + switch n := node.(type) { + case *sql2.Call: + for i := range n.Args { + if err := v.checkExpr(&n.Args[i]); err != nil { + return nil, nil, err + } + } + return nil, node, nil // skip + case *sql2.Ident: + if node, err = v.visitIdent(n); err != nil { + return nil, nil, err + } + return nil, node, nil + case *sql2.QualifiedRef: + if node, err = v.visitQualifiedRef(n); err != nil { + return nil, nil, err + } + return nil, node, nil + default: + return v, node, nil + } +} + +func (v *sqlExprTypeChecker) visitIdent(ident *sql2.Ident) (sql2.Node, error) { + indexName, err := statementTableName(v.stmt) + if err != nil { + return nil, err + } + + // Convert to a table qualified reference and validate through ref visit function. + return v.visitQualifiedRef(&sql2.QualifiedRef{ + Table: &sql2.Ident{Name: indexName}, + Column: &sql2.Ident{Name: ident.Name}, + }) +} + +func (v *sqlExprTypeChecker) visitQualifiedRef(ref *sql2.QualifiedRef) (sql2.Node, error) { + idx := v.holder.Index(ref.Table.Name) + if idx == nil { + return nil, newNotFoundError(ErrIndexNotFound, ref.Table.Name) + } + + switch name := ref.Column.Name; name { + case "_id": + ref.DataType = sql2.DataTypeInt + default: + field := idx.Field(ref.Column.Name) + if field == nil { + return nil, newNotFoundError(ErrFieldNotFound, ref.Column.Name) + } + ref.DataType = fieldSQLDataType(field) + } + + return ref, nil +} + +func (v *sqlExprTypeChecker) checkExpr(node *sql2.Expr) error { + if expr, err := sql2.Walk(&sqlExprTypeChecker{ + holder: v.holder, + stmt: v.stmt, + }, *node); err != nil { + return err + } else if expr != nil { + *node = expr.(sql2.Expr) + } else { + *node = nil + } + return nil +} + +func (v *sqlExprTypeChecker) VisitEnd(node sql2.Node) (sql2.Node, error) { return node, nil } + +func fieldSQLDataType(f *Field) string { + if f.Keys() { + return sql2.DataTypeText + } + + switch f.Type() { + case FieldTypeInt, FieldTypeMutex, FieldTypeSet: + return sql2.DataTypeInt + case FieldTypeBool: + return sql2.DataTypeBool + case FieldTypeDecimal: + return sql2.DataTypeDecimal + case FieldTypeTime, FieldTypeTimestamp: + return sql2.DataTypeTimestamp + default: + return "" + } +} + type Stmt struct { node StmtNode } @@ -413,10 +813,16 @@ func (rs *StmtRows) Err() error { return nil } -func (rs *StmtRows) Columns() []string { +func (rs *StmtRows) Columns() []*StmtColumn { return rs.node.Columns() } +/* +func (rs *StmtRows) Row() int64 { + return rs.node.Row()[0].(int64) +} +*/ + func (rs *StmtRows) Next() bool { if rs.err != nil { return false @@ -463,6 +869,15 @@ func (rs *StmtRows) Scan(dst ...interface{}) error { // Copy row value to scan destination. switch v := row[i].(type) { + case bool: + switch p := dst[i].(type) { + case *bool: + *p = v + case *interface{}: + *p = v + default: + return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i) + } case int64: switch p := dst[i].(type) { case *int: @@ -478,6 +893,48 @@ func (rs *StmtRows) Scan(dst ...interface{}) error { default: return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i) } + case uint64: + switch p := dst[i].(type) { + case *int: + *p = int(v) + case *int64: + *p = int64(v) + case *uint: + *p = uint(v) + case *uint64: + *p = uint64(v) + case *interface{}: + *p = v + default: + return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i) + } + case []uint64: + switch p := dst[i].(type) { + case *[]uint64: + *p = []uint64(v) + case *interface{}: + *p = joinUint64Slice(v) + default: + return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i) + } + case string: + switch p := dst[i].(type) { + case *string: + *p = v + case *interface{}: + *p = v + default: + return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i) + } + case []string: + switch p := dst[i].(type) { + case *[]string: + *p = []string(v) + case *interface{}: + *p = strings.Join(v, ",") + default: + return fmt.Errorf("cannot scan %T value into %T destination at index %d", v, p, i) + } default: return fmt.Errorf("unexpected %T value at index %d", v, i) } @@ -514,6 +971,11 @@ func (r *StmtRow) Err() error { return r.err } +type StmtColumn struct { + Name string + Type string +} + type StmtNode interface { // Initializes the node to its start. First(ctx context.Context) error @@ -525,7 +987,7 @@ type StmtNode interface { Row() []interface{} // Returns column definitions for the node. - Columns() []string + Columns() []*StmtColumn // Returns a reference to the value register for a named column. // Lookup(table, column string) (interface{}, error) @@ -537,39 +999,47 @@ var _ StmtNode = (*ExtractNode)(nil) type ExtractNode struct { executor *executor indexName string - columns []string - aliases []string + srcs []string + columns []*StmtColumn + mapping []int // map of output column indices to source column indices cond *pql.Call result []ExtractedTableColumn row []interface{} } -func NewExtractNode(executor *executor, indexName string, columns, aliases []string, cond *pql.Call) *ExtractNode { +func NewExtractNode(executor *executor, indexName string, srcs []string, columns []*StmtColumn, cond *pql.Call) *ExtractNode { if cond == nil { cond = &pql.Call{Name: "All"} } - // Ensure ID column is always the first column. - if len(columns) > 0 && columns[0] != "_id" { - columns = append([]string{"_id"}, columns...) - aliases = append([]string{"_id"}, aliases...) - } + // Determine mapping between result elements & columns. + // We'll exclude "_id" from the source columns here as well. + mapping := make([]int, len(columns)) + srcs2 := make([]string, 0, len(srcs)) + for i := range mapping { + if srcs[i] == "_id" { + mapping[i] = -1 + continue + } - // TODO: Move "_id" column to the first position if it is specified later on in column list. + mapping[i] = len(srcs2) + srcs2 = append(srcs2, srcs[i]) + } return &ExtractNode{ executor: executor, indexName: indexName, - columns: columns, // source column names - aliases: aliases, // external column alias + srcs: srcs2, // source column names (excluding "id") + columns: columns, // external column alias + mapping: mapping, cond: cond, - row: make([]interface{}, len(columns)), + row: make([]interface{}, len(srcs)), } } -func (n *ExtractNode) Columns() []string { - return n.aliases +func (n *ExtractNode) Columns() []*StmtColumn { + return n.columns } func (n *ExtractNode) First(ctx context.Context) error { @@ -578,10 +1048,12 @@ func (n *ExtractNode) First(ctx context.Context) error { } func (n *ExtractNode) Next(ctx context.Context) error { + // Fetch results if we haven't yet. if err := n.init(ctx); err != nil { return err } + // Exit if no result rows remain. if len(n.result) == 0 { for i := range n.row { n.row[i] = nil @@ -589,18 +1061,25 @@ func (n *ExtractNode) Next(ctx context.Context) error { return sql.ErrNoRows } - // Copy ID value to current row. - result := n.result[0] - if result.Column.Keyed { - n.row[0] = result.Column.Key - } else { - n.row[0] = int64(result.Column.ID) + // Map result elements to row elements. + for i, index := range n.mapping { + result := n.result[0] + + // Map row array element to position in result row. + if index >= 0 { + n.row[i] = result.Rows[index] + continue + } + + // Otherwise use ID for value. + if result.Column.Keyed { + n.row[i] = result.Column.Key + } else { + n.row[i] = int64(result.Column.ID) + } } - // Copy values to current row. - for i, v := range result.Rows { - n.row[i+1] = v - } + // Move to next result element. n.result = n.result[1:] return nil @@ -614,11 +1093,11 @@ func (n *ExtractNode) init(ctx context.Context) error { // Generate PQL query with all specified rows. // Skip first column as it is the ID column. call := &pql.Call{Name: "Extract", Children: []*pql.Call{n.cond}} - for _, column := range n.columns[1:] { + for _, src := range n.srcs { call.Children = append(call.Children, &pql.Call{ Name: "Rows", - Args: map[string]interface{}{"field": column}, + Args: map[string]interface{}{"field": src}, }, ) } @@ -649,28 +1128,25 @@ var _ StmtNode = (*CountNode)(nil) // CountNode executes a COUNT(*) against a FeatureBase index and returns a single row. type CountNode struct { - executor *executor - indexName string - columnName string - cond *pql.Call // conditional + executor *executor + indexName string + column *StmtColumn + call *pql.Call row []interface{} } -func NewCountNode(executor *executor, indexName string, columnName string, cond *pql.Call) *CountNode { - if cond == nil { - cond = &pql.Call{Name: "All"} - } +func NewCountNode(executor *executor, indexName string, column *StmtColumn, call *pql.Call) *CountNode { return &CountNode{ - executor: executor, - indexName: indexName, - columnName: columnName, - cond: cond, + executor: executor, + indexName: indexName, + column: column, + call: call, } } -func (n *CountNode) Columns() []string { - return []string{n.columnName} +func (n *CountNode) Columns() []*StmtColumn { + return []*StmtColumn{n.column} } func (n *CountNode) First(ctx context.Context) error { @@ -683,13 +1159,7 @@ func (n *CountNode) Next(ctx context.Context) error { return sql.ErrNoRows } - q := &pql.Query{ - Calls: []*pql.Call{ - {Name: "Count", Children: []*pql.Call{n.cond}}, - }, - } - - result, err := n.executor.Execute(ctx, n.indexName, q, nil, nil) + result, err := n.executor.Execute(ctx, n.indexName, &pql.Query{Calls: []*pql.Call{n.call}}, nil, nil) if err != nil { return err } @@ -699,3 +1169,174 @@ func (n *CountNode) Next(ctx context.Context) error { } func (n *CountNode) Row() []interface{} { return n.row } + +// GroupByNode executes an aggregate with a GROUP BY against a FeatureBase index. +type GroupByNode struct { + executor *executor + indexName string + groupByCols []string + columns []*StmtColumn + mapping []int + aggregate *pql.Call + cond *pql.Call + + result *GroupCounts + index int + + row []interface{} +} + +func NewGroupByNode(executor *executor, indexName string, resultCols, groupByCols []string, columns []*StmtColumn, aggregate, cond *pql.Call) *GroupByNode { + // Map result columns to output columns. + mapping := make([]int, len(columns)) + for i := range mapping { + if resultCols[i] == "_aggregate" { + mapping[i] = -1 + continue + } + + mapping[i] = stringSliceIndex(groupByCols, resultCols[i]) + } + + return &GroupByNode{ + executor: executor, + indexName: indexName, + groupByCols: groupByCols, + columns: columns, + mapping: mapping, + aggregate: aggregate, + cond: cond, + row: make([]interface{}, len(columns)), + } +} + +func (n *GroupByNode) Columns() []*StmtColumn { + return n.columns +} + +func (n *GroupByNode) First(ctx context.Context) error { + n.result = nil + return nil +} + +func (n *GroupByNode) Next(ctx context.Context) (err error) { + // Fetch resultset if it doesn't exist yet. + if n.result == nil { + if n.result, err = n.fetch(ctx); err != nil { + return err + } + } + + // Exit if no more rows exist. + if n.index >= len(n.result.groups) { + return sql.ErrNoRows + } + + // Copy results into current row. + group := n.result.groups[n.index] + n.index++ + + for i, index := range n.mapping { + // Assign aggregate to unmapped column. + if index == -1 { + if n.aggregate != nil { + n.row[i] = int64(group.Agg) + } else { + n.row[i] = int64(group.Count) + } + continue + } + + // Otherwise map from group value to result column index. + g := group.Group[index] + if g.Value != nil { + n.row[i] = *g.Value + } else if g.RowKey != "" { + n.row[i] = g.RowKey + } else { + n.row[i] = int64(g.RowID) + } + } + + return nil +} + +// fetch executes a call to compute the PQL results. +func (n *GroupByNode) fetch(ctx context.Context) (*GroupCounts, error) { + call := &pql.Call{ + Name: "GroupBy", + Args: map[string]interface{}{}, + } + + // Choose fields to group by. + for _, name := range n.groupByCols { + call.Children = append(call.Children, &pql.Call{ + Name: "Rows", Args: map[string]interface{}{"_field": name}, + }) + } + + // Apply filter & aggregate, if set. + if n.aggregate != nil { + call.Args["aggregate"] = n.aggregate + } + if n.cond != nil { + call.Args["filter"] = n.cond + } + + result, err := n.executor.Execute(ctx, n.indexName, &pql.Query{Calls: []*pql.Call{call}}, nil, nil) + if err != nil { + return nil, err + } + return result.Results[0].(*GroupCounts), nil +} + +func (n *GroupByNode) Row() []interface{} { return n.row } + +// statementTableName returns the table name for a single table SELECT statement. +// +// NOTE: This function is only temporary until we support more source types. +func statementTableName(stmt sql2.Statement) (string, error) { + switch stmt := stmt.(type) { + case *sql2.SelectStatement: + return sourceTableName(stmt.Source) + default: + return "", fmt.Errorf("statement not currently supported") + } +} + +func sourceTableName(source sql2.Source) (string, error) { + switch source := source.(type) { + case *sql2.JoinClause: + return "", fmt.Errorf("joins are not currently supported") + case *sql2.ParenSource: + return "", fmt.Errorf("parenthesized source is not currently supported") + case *sql2.QualifiedTableName: + return sql2.IdentName(source.Name), nil + case *sql2.SelectStatement: + return "", fmt.Errorf("sub-selects are not currently supported") + default: + return "", fmt.Errorf("unexpected source type: %T", source) + } +} + +// stringSliceIndex returns position of v in a. Returns -1 if not found. +func stringSliceIndex(a []string, v string) int { + for i := range a { + if a[i] == v { + return i + } + } + return -1 +} + +func joinUint64Slice(a []uint64) string { + b := []byte("[") + for i, v := range a { + b = strconv.AppendUint(b, v, 10) + if i < len(a)-1 { + b = append(b, ',') + } + } + b = append(b, ']') + return string(b) +} diff --git a/planner_test.go b/planner_test.go index bb9ff3bd0..8694b724c 100644 --- a/planner_test.go +++ b/planner_test.go @@ -210,60 +210,308 @@ func TestPlanner_Select(t *testing.T) { } t.Run("UnqualifiedColumns", func(t *testing.T) { - results := mustQueryRows(t, c.GetNode(0).Server, `SELECT _id, a, b FROM i0`) - if diff := cmp.Diff(results, [][]interface{}{ - {int64(1), int64(10), int64(100)}, - {int64(2), int64(20), int64(200)}, - }); diff != "" { + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT a, b, _id FROM i0`) + if diff := cmp.Diff([][]interface{}{ + {int64(10), int64(100), int64(1)}, + {int64(20), int64(200), int64(2)}, + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "a", Type: "INT"}, + {Name: "b", Type: "INT"}, + {Name: "_id", Type: "INT"}, + }, columns); diff != "" { t.Fatal(diff) } }) t.Run("QualifiedColumns", func(t *testing.T) { - results := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0._id, i0.a, i0.b FROM i0`) - if diff := cmp.Diff(results, [][]interface{}{ + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0._id, i0.a, i0.b FROM i0`) + if diff := cmp.Diff([][]interface{}{ {int64(1), int64(10), int64(100)}, {int64(2), int64(20), int64(200)}, - }); diff != "" { + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "_id", Type: "INT"}, + {Name: "a", Type: "INT"}, + {Name: "b", Type: "INT"}, + }, columns); diff != "" { t.Fatal(diff) } }) t.Run("UnqualifiedStar", func(t *testing.T) { - results := mustQueryRows(t, c.GetNode(0).Server, `SELECT * FROM i0`) - if diff := cmp.Diff(results, [][]interface{}{ + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT * FROM i0`) + if diff := cmp.Diff([][]interface{}{ {int64(1), int64(10), int64(100)}, {int64(2), int64(20), int64(200)}, - }); diff != "" { + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "_id", Type: "INT"}, + {Name: "a", Type: "INT"}, + {Name: "b", Type: "INT"}, + }, columns); diff != "" { t.Fatal(diff) } }) t.Run("QualifiedStar", func(t *testing.T) { - results := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0.* FROM i0`) - if diff := cmp.Diff(results, [][]interface{}{ + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0.* FROM i0`) + if diff := cmp.Diff([][]interface{}{ {int64(1), int64(10), int64(100)}, {int64(2), int64(20), int64(200)}, - }); diff != "" { + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "_id", Type: "INT"}, + {Name: "a", Type: "INT"}, + {Name: "b", Type: "INT"}, + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("NoIdentifier", func(t *testing.T) { + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT a, b FROM i0`) + if diff := cmp.Diff([][]interface{}{ + {int64(10), int64(100)}, + {int64(20), int64(200)}, + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "a", Type: "INT"}, + {Name: "b", Type: "INT"}, + }, columns); diff != "" { t.Fatal(diff) } }) t.Run("ErrFieldNotFound", func(t *testing.T) { - stmt, err := c.GetNode(0).Server.PlanSQL(context.Background(), `SELECT xyz FROM i0`) - if err != nil { - t.Fatal(err) - } - defer stmt.Close() - - var xyz interface{} - if err := stmt.QueryRowContext(context.Background()).Scan(&xyz); err == nil || !strings.Contains(err.Error(), `xyz: field not found`) { + _, err := c.GetNode(0).Server.PlanSQL(context.Background(), `SELECT xyz FROM i0`) + if err == nil || !strings.Contains(err.Error(), `xyz: field not found`) { t.Fatalf("unexpected error: %v", err) } }) } -func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{} { +func TestPlanner_GroupBy(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + + i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true}) + if err != nil { + t.Fatal(err) + } + defer i0.Close() + + if _, err := i0.CreateField("x"); err != nil { + t.Fatal(err) + } else if _, err := i0.CreateField("y", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } else if _, err := i0.CreateField("z", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + // Populate with data. + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i0", + Query: ` + Set(1, x=10) + Set(1, x=20) + Set(1, y=100) + Set(1, z=500) + + Set(2, x=10) + Set(2, y=200) + Set(2, z=500) + + Set(3, x=20) + Set(3, z=600) + `}); err != nil { + t.Fatal(err) + } + + t.Run("Count", func(t *testing.T) { + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*), x FROM i0 GROUP BY x`) + if diff := cmp.Diff([][]interface{}{ + {int64(2), int64(10)}, + {int64(2), int64(20)}, + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "count", Type: "INT"}, + {Name: "x", Type: "INT"}, + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("DistinctCount", func(t *testing.T) { + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(DISTINCT z), x FROM i0 GROUP BY x`) + if diff := cmp.Diff([][]interface{}{ + {int64(1), int64(10)}, + {int64(2), int64(20)}, + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "count", Type: "INT"}, + {Name: "x", Type: "INT"}, + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("Sum", func(t *testing.T) { + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT sum(y), x FROM i0 GROUP BY x`) + if diff := cmp.Diff([][]interface{}{ + {int64(300), int64(10)}, + {int64(100), int64(20)}, + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "sum", Type: "INT"}, + {Name: "x", Type: "INT"}, + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("ReorderColumns", func(t *testing.T) { + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT x, COUNT(*) FROM i0 GROUP BY x`) + if diff := cmp.Diff([][]interface{}{ + {int64(10), int64(2)}, + {int64(20), int64(2)}, + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "x", Type: "INT"}, + {Name: "count", Type: "INT"}, + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("NoResultColumn", func(t *testing.T) { + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 GROUP BY x`) + if diff := cmp.Diff([][]interface{}{ + {int64(2)}, + {int64(2)}, + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "count", Type: "INT"}, + }, columns); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestPlanner_InnerJoin(t *testing.T) { + c := test.MustRunCluster(t, 1) + defer c.Close() + + i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true}) + if err != nil { + t.Fatal(err) + } + defer i0.Close() + + if _, err := i0.CreateField("a", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + i1, err := c.GetHolder(0).CreateIndex("i1", pilosa.IndexOptions{TrackExistence: true}) + if err != nil { + t.Fatal(err) + } + defer i1.Close() + + if _, err := i1.CreateField("parentid", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } else if _, err := i1.CreateField("x", pilosa.OptFieldTypeInt(0, 1000)); err != nil { + t.Fatal(err) + } + + // Populate with data. + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i0", + Query: ` + Set(1, a=10) + Set(2, a=20) + Set(3, a=30) + `}); err != nil { + t.Fatal(err) + } + + if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{ + Index: "i1", + Query: ` + Set(1, parentid=1) + Set(1, x=100) + + Set(2, parentid=1) + Set(2, x=200) + + Set(3, parentid=2) + Set(3, x=300) + `}); err != nil { + t.Fatal(err) + } + + t.Run("Count", func(t *testing.T) { + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid`) + if diff := cmp.Diff([][]interface{}{ + {int64(2)}, + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "count", Type: "INT"}, + }, columns); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("CountWithParentCondition", func(t *testing.T) { + results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid WHERE i0.a = 10`) + if diff := cmp.Diff([][]interface{}{ + {int64(1)}, + }, results); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff([]*pilosa.StmtColumn{ + {Name: "count", Type: "INT"}, + }, columns); diff != "" { + t.Fatal(diff) + } + }) +} + +func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) (results [][]interface{}, columns []*pilosa.StmtColumn) { tb.Helper() stmt, err := svr.PlanSQL(context.Background(), q) @@ -277,7 +525,7 @@ func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{} tb.Fatal(err) } - results := make([][]interface{}, 0) + results = make([][]interface{}, 0) for rows.Next() { result := make([]interface{}, len(rows.Columns())) @@ -297,5 +545,5 @@ func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{} tb.Fatal(err) } - return results + return results, rows.Columns() } diff --git a/rbf/db.go b/rbf/db.go index 92efffd72..f1623ded8 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -274,7 +274,7 @@ func (db *DB) Close() (err error) { // Close writer handler. if db.file != nil { - err = db.file.Sync() + err = db.fsync(db.file) if err != nil { return } diff --git a/roaring/roaring.go b/roaring/roaring.go index 9567b37fb..e35487a3e 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2997,6 +2997,7 @@ func ArrayCountRange(array []uint16, start, end int32) (n int32) { return n } +// BitmapCountRange counts bits set in [start,end). func BitmapCountRange(bitmap []uint64, start, end int32) int32 { if roaringParanoia { if start > end { @@ -3033,15 +3034,16 @@ func BitmapCountRange(bitmap []uint64, start, end int32) int32 { } func callbackBits(w uint64, base uint16, fn func(uint16)) { - bit := uint16(0) for w != 0 { - trail := bits.TrailingZeros64(w) - bit += uint16(trail) - w >>= (trail + 1) - fn(base + bit) + trail := uint16(bits.TrailingZeros64(w)) + fn(base + trail) + base += trail + 1 + w >>= trail + 1 } } +// bitmapCallbackRange calls the provided function for every bit set in +// bitmap in the range [start,end). func bitmapCallbackRange(bitmap []uint64, start, end int32, fn func(uint16)) { if roaringParanoia { if start > end { @@ -3051,11 +3053,30 @@ func bitmapCallbackRange(bitmap []uint64, start, end int32, fn func(uint16)) { i, j := start/64, end/64 // Special case when start and end fall in the same word. if i == j { - offi, offj := uint(start%64), uint(64-end%64) - w := (bitmap[i] >> offi) << (offj + offi) + // So, we want to know the offsets. For instance, if start and end + // are 65 and 69, we might want i=1, offi=1, j=1, offj=5. Then we + // compute masks from offi (masking out 0x1, or (1<> offi << offi" to trim the lowest offi + // bits, and "x << (64-offj) >> (64-offj)" to trim all but the + // lowest offj bits. + // + // We can then simplify slightly further: we use the inverted value + // as offj, and compute (w << offi) >> (offi + offj) << offi. + // + // But wait, you ask. What if offi+offj is too large! Well, then + // start and end were in the wrong order. We have 0 <= i <= j < 64. + // If x+i > 64, then x > (64-i). Thus, if (64-j)+i > 64, it + // follows that (64-j) > (64-i). So they'd have been in the wrong order. + // In which case, we correctly yield a value of (0 << offi), or 0, + // because nothing is between them. + offi, offj := uint(start%64), uint(64-(end%64)) + w := (bitmap[i] << offj) >> (offi + offj) << offi if w != 0 { callbackBits(w, uint16(i)*64, fn) } + return } // Count partial starting word. @@ -4492,7 +4513,7 @@ func intersectionCallbackArrayArray(a, b *Container, fn func(uint16)) { } if (na << 2) < nb { for _, va := range ca { - for cb[0] < va { + if cb[0] < va { // try to skip ahead a bit faster for len(cb) > 7 && cb[7] < va { cb = cb[8:] @@ -4530,19 +4551,21 @@ func intersectionCallbackArrayRun(a, b *Container, fn func(uint16)) { na, nb := len(array), len(runs) for i, j := 0, 0; i < na && j < nb; { va, vb := array[i], runs[j] - if va < vb.Start { - i++ - } else if va >= vb.Start && va <= vb.Last { - i++ - fn(va) - } else if va > vb.Last { + if va > vb.Last { j++ + continue + } + // If we got here, va is either before or in the current run, + // so we're definitely done with this member of the array. + i++ + if va >= vb.Start { + fn(va) } } } func intersectionCallbackRunRun(a, b *Container, fn func(uint16)) { - statsHit("intersectionCount/RunRun") + statsHit("intersectionCallback/RunRun") ra, rb := a.runs(), b.runs() na, nb := len(ra), len(rb) for i, j := 0, 0; i < na && j < nb; { @@ -4582,14 +4605,14 @@ func intersectionCallbackRunRun(a, b *Container, fn func(uint16)) { } func intersectionCallbackBitmapRun(a, b *Container, fn func(uint16)) { - statsHit("intersectionCount/BitmapRun") + statsHit("intersectionCallback/BitmapRun") for _, iv := range b.runs() { bitmapCallbackRange(a.bitmap(), int32(iv.Start), int32(iv.Last)+1, fn) } } func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) { - statsHit("intersectionCount/ArrayBitmap") + statsHit("intersectionCallback/ArrayBitmap") bitmap := b.bitmap() ln := len(bitmap) for _, val := range a.array() { @@ -4598,14 +4621,14 @@ func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) { break } off := val % 64 - if (bitmap[i]>>off) & 1 != 0 { + if (bitmap[i]>>off)&1 != 0 { fn(val) } } } func intersectionCallbackBitmapBitmap(a, b *Container, fn func(uint16)) { - statsHit("intersectionCount/BitmapBitmap") + statsHit("intersectionCallback/BitmapBitmap") ab, bb := a.bitmap(), b.bitmap() for i := range ab { w := ab[i] & bb[i] diff --git a/roaring/roaring_internal_test.go b/roaring/roaring_internal_test.go index ce6f61a7d..c23adc6f4 100644 --- a/roaring/roaring_internal_test.go +++ b/roaring/roaring_internal_test.go @@ -4636,3 +4636,103 @@ func TestContainer_unionInPlace_ArrayUnionRun(t *testing.T) { } } } + +func TestContainerCallback(t *testing.T) { + containers, err := InitContainerArchetypes() + if err != nil { + t.Fatalf("creating containers: %v", err) + } + got := make([]uint16, 65536) + hit := func(u uint16) { + got = append(got, u) + } + var expected []uint16 + // complain() wraps up some pretty-printing logic for this, + // but note also the closure trapping expected/got so we can + // just refer to them without passing them in. + complain := func(t *testing.T, msg string, args ...interface{}) { + l1 := len(expected) + l2 := len(got) + dotdot1 := "" + dotdot2 := "" + if l1 > 8 { + expected = expected[:8] + dotdot1 = "..." + } + if l2 > 8 { + got = got[:8] + dotdot2 = "..." + } + t.Fatalf("%s: expected %d%s, got %d%s", fmt.Sprintf(msg, args...), expected, dotdot1, got, dotdot2) + } + for t1, ci := range containers { + t.Run(ContainerArchetypeNames[t1], func(t *testing.T) { + for _, c1 := range ci { + got = got[:0] + expected = c1.Slice() + containerCallback(c1, hit) + if len(got) != len(expected) { + complain(t, "wrong length (%d vs %d)", len(expected), len(got)) + } + for i := range got { + if got[i] != expected[i] { + complain(t, "element %d differs: expected %d, got %d", i, expected[i], got[i]) + } + } + } + }) + } +} + +func TestIntersectionCallback(t *testing.T) { + containers, err := InitContainerArchetypes() + if err != nil { + t.Fatalf("creating containers: %v", err) + } + got := make([]uint16, 65536) + hit := func(u uint16) { + got = append(got, u) + } + var expected []uint16 + // complain() wraps up some pretty-printing logic for this, + // but note also the closure trapping expected/got so we can + // just refer to them without passing them in. + complain := func(t *testing.T, msg string, args ...interface{}) { + l1 := len(expected) + l2 := len(got) + dotdot1 := "" + dotdot2 := "" + if l1 > 8 { + expected = expected[:8] + dotdot1 = "..." + } + if l2 > 8 { + got = got[:8] + dotdot2 = "..." + } + t.Fatalf("%s: expected %d%s, got %d%s", fmt.Sprintf(msg, args...), expected, dotdot1, got, dotdot2) + } + for t1, ci := range containers { + for t2, cj := range containers { + t.Run(fmt.Sprintf("%s-%s", ContainerArchetypeNames[t1], ContainerArchetypeNames[t2]), func(t *testing.T) { + for _, c1 := range ci { + for _, c2 := range cj { + got = got[:0] + expectedContainer := intersect(c1, c2) + expected = expectedContainer.Slice() + intersectionCallback(c1, c2, hit) + if len(got) != len(expected) { + complain(t, "wrong length (%d vs %d)", len(expected), len(got)) + } + for i := range got { + if got[i] != expected[i] { + complain(t, "element %d differs: expected %d, got %d", i, expected[i], got[i]) + } + } + } + } + + }) + } + } +} diff --git a/server.go b/server.go index c9b4d8d5e..fdca4dc39 100644 --- a/server.go +++ b/server.go @@ -339,6 +339,9 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption { func OptServerStorageConfig(cfg *storage.Config) ServerOption { return func(s *Server) error { s.holderConfig.StorageConfig = cfg + // For historical reasons, RBF's config can ignore the storage config + // in some cases. + s.holderConfig.RBFConfig.FsyncEnabled = s.holderConfig.StorageConfig.FsyncEnabled return nil } } @@ -437,7 +440,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { confirmDownRetries: defaultConfirmDownRetries, confirmDownSleep: defaultConfirmDownSleep, - resetTranslationSyncCh: make(chan struct{}), + resetTranslationSyncCh: make(chan struct{}, 1), logger: logger.NopLogger, } @@ -569,11 +572,6 @@ func (s *Server) Open() error { log.Println(errors.Wrap(err, "logging startup")) } - // Start background process listening for translation - // sync resets. - s.wg.Add(1) - go func() { defer s.wg.Done(); s.monitorResetTranslationSync() }() - // Start DisCo. ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) defer cancel() @@ -612,6 +610,12 @@ func (s *Server) Open() error { s.syncer.Closing = s.closing s.syncer.Stats = s.holder.Stats.WithTags("component:HolderSyncer") + // Start background process listening for translation + // sync resets. + s.wg.Add(1) + go func() { defer s.wg.Done(); s.monitorResetTranslationSync() }() + go func() { _ = s.translationSyncer.Reset() }() + // Open holder. func() { s.holder.startMsgsMu.Lock() diff --git a/server/config.go b/server/config.go index b96a5be1f..dc6ae08ae 100644 --- a/server/config.go +++ b/server/config.go @@ -200,6 +200,9 @@ type Config struct { // Setting this to 0 disables the limit. // This mostly exists because other DBs seem to have it. ConnectionLimit uint16 `toml:"max-connections"` + // SqlVersion is which type of sqlhandling to be applied. + // The constant SqlV2 can be used to try the new experimental Molecula SQL handling + SqlVersion uint16 `toml:"sql-version"` } `toml:"postgres"` // Storage.Backend determines which Tx implementation the holder/Index will diff --git a/server/pg.go b/server/pg.go index 0f03e7dfb..533ca0b43 100644 --- a/server/pg.go +++ b/server/pg.go @@ -25,9 +25,11 @@ import ( "strings" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/pg" + + //"github.com/molecula/featurebase/v2/pg" "github.com/molecula/featurebase/v2/pql" pb "github.com/molecula/featurebase/v2/proto" @@ -43,14 +45,20 @@ type PostgresServer struct { s pg.Server stop context.CancelFunc } +type SqlVersion uint16 + +const ( + SqlV1 SqlVersion = 0 + SqlV2 SqlVersion = 2 +) // NewPostgresServer creates a postgres server. -func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config) *PostgresServer { +func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config, sqlVersion SqlVersion) *PostgresServer { return &PostgresServer{ api: api, logger: logger, s: pg.Server{ - QueryHandler: NewPostgresHandler(api, logger), + QueryHandler: NewPostgresHandler(api, logger, sqlVersion), TypeEngine: pg.PrimitiveTypeEngine{}, StartupTimeout: 5 * time.Second, ReadTimeout: 10 * time.Second, @@ -66,11 +74,12 @@ func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config) * } // NewPostgresHandler creates a postgres query handler wrapping the pilosa API. -func NewPostgresHandler(api *pilosa.API, logger logger.Logger) pg.QueryHandler { - return &queryDecodeHandler{ - child: &pilosaQueryHandler{ - api: api, - logger: logger, +func NewPostgresHandler(api *pilosa.API, logger logger.Logger, sqlVersion SqlVersion) pg.QueryHandler { + return &QueryDecodeHandler{ + Child: &PilosaQueryHandler{ + Api: api, + logger: logger, + sqlVersion: sqlVersion, }, } } @@ -107,6 +116,10 @@ func (s *PostgresServer) Close() error { return s.eg.Wait() } +func (s *PostgresServer) GetAPI() *pilosa.API { + return s.api +} + type pgPQLQuery struct { index string query string @@ -135,9 +148,10 @@ func pgDecodePQL(str string) (q pg.Query, err error) { }, nil } -type pilosaQueryHandler struct { - api *pilosa.API - logger logger.Logger +type PilosaQueryHandler struct { + Api *pilosa.API + logger logger.Logger + sqlVersion SqlVersion } func pgWriteRow(w pg.QueryResultWriter, row *pilosa.Row) error { @@ -322,6 +336,78 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error return nil } +func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error { + //TODO(twg) writeHeader + //TODO(twg) writeColumns + first := true + var data []string + var err error + for rows.Next() { + if first { + columns := rows.Columns() + //TODO (twg) types:=rows.Types() + headers := make([]pg.ColumnInfo, len(columns)) + for i, column := range columns { + headers[i] = pg.ColumnInfo{ + Name: column.Name, + Type: pg.TypeCharoid, //TODO(twg) types[i] + } + } + err := w.WriteHeader(headers...) + if err != nil { + return err + } + + data = make([]string, len(headers)) + first = false + } + result := make([]interface{}, len(rows.Columns())) + // Create list of scan destination pointers. + dsts := make([]interface{}, len(result)) + for i := range result { + dsts[i] = &result[i] + } + + if err := rows.Scan(dsts...); err != nil { + return err + } + //TODO(twg) conversion should be happening in Scan as described in https://pkg.go.dev/database/sql + // .... + // Scan also converts between string and numeric types, as long as no information + // would be lost. While Scan stringifies all numbers scanned from numeric database columns into *string, + // scans into numeric types are checked for overflow. For example, a float64 with value 300 or a string + // with value "300" can scan into a uint16, but not into a uint8, though float64(255) or "255" can scan + // into a uint8. One exception is that scans of some float64 numbers to strings may lose information when stringifying. + // In general, scan floating point columns into *float64. + // ... + for i, col := range dsts { + var v string + switch col := col.(type) { + case nil: + v = "null" + // + //v = strconv.FormatUint(col.Uint64Val, 10) + case *interface{}: + v = fmt.Sprintf("%v", *col) + default: + return errors.Errorf("unable to process value of type %T", col) + } + + data[i] = v + } + + err = w.WriteRowText(data...) + if err != nil { + return err + } + + } + if err := rows.Err(); err != nil { + return err + } + return nil +} + func pgWriteRowser(w pg.QueryResultWriter, result pb.ToRowser) error { var data []string return result.ToRows(func(row *pb.RowResponse) error { @@ -395,6 +481,8 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { return pgWriteGroupCount(w, result) case pb.ToRowser: // we should avoid protobuf where we can... return pgWriteRowser(w, result) + case *pilosa.StmtRows: // we should avoid protobuf where we can... + return pgWriteStmtRows(w, result) case uint64: err := w.WriteHeader(pg.ColumnInfo{ Name: "count", @@ -449,10 +537,10 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error { } } -func (pqh *pilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error { +func (pqh *PilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error { switch q := q.(type) { case pgPQLQuery: - resp, err := pqh.api.Query(ctx, &pilosa.QueryRequest{ + resp, err := pqh.Api.Query(ctx, &pilosa.QueryRequest{ Index: q.index, Query: q.query, }) @@ -465,22 +553,36 @@ func (pqh *pilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResult return errors.Wrap(pgWriteResult(w, resp.Results[0]), "writing query result") case pg.SimpleQuery: - resp, err := execSQL(ctx, pqh.api, pqh.logger, string(q)) - if err != nil { - return errors.Wrap(err, "executing query") + if pqh.sqlVersion == SqlV2 { + stmt, err := pqh.Api.Plan(ctx, string(q)) + if err != nil { + return err + } + resp, err := stmt.QueryContext(ctx) + if err != nil { + return err + } + return errors.Wrap(pgWriteResult(w, resp), "writing sql2 query result") + //version 2.0 + } else { + //version 1.0 + resp, err := execSQL(ctx, pqh.Api, pqh.logger, string(q)) + if err != nil { + return errors.Wrap(err, "executing query") + } + return errors.Wrap(pgWriteResult(w, resp), "writing query result") } - return errors.Wrap(pgWriteResult(w, resp), "writing query result") default: return errors.Errorf("query type %T not yet supported (query: %s)", q, q) } } -type queryDecodeHandler struct { - child pg.QueryHandler +type QueryDecodeHandler struct { + Child pg.QueryHandler } -func (qdh *queryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error { +func (qdh *QueryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error { switch qv := q.(type) { case pg.SimpleQuery: if strings.HasPrefix(string(qv), "[") { @@ -492,5 +594,33 @@ func (qdh *queryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResult } } - return qdh.child.HandleQuery(ctx, w, q) + return qdh.Child.HandleQuery(ctx, w, q) +} +func (qdh *QueryDecodeHandler) Version() string { + return qdh.Child.Version() +} + +func (pqh *PilosaQueryHandler) Version() string { + if pqh.sqlVersion > 0 { + return "v2" + } + return "v1" +} +func (pqh *PilosaQueryHandler) HandleSchema(ctx context.Context, portal *pg.Portal) error { + schema, err := pqh.Api.Schema(context.Background(), false) + if err != nil { + return err + } + for _, ii := range schema { + dataRow, err := portal.Encoder.TextRow("featurebase", ii.Name) + if err != nil { + return err + } + portal.Add(dataRow) + } + return nil +} + +func (qdh *QueryDecodeHandler) HandleSchema(ctx context.Context, portal *pg.Portal) error { + return qdh.Child.HandleSchema(ctx, portal) } diff --git a/server/pg_test.go b/server/pg_test.go index 012986afc..bd8a88521 100644 --- a/server/pg_test.go +++ b/server/pg_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/logger" "github.com/molecula/featurebase/v2/pg" "github.com/molecula/featurebase/v2/pg/pgtest" @@ -33,7 +33,7 @@ func TestPostgresHandler(t *testing.T) { m := test.RunCommand(t) defer m.Close() - pgh := server.NewPostgresHandler(m.API, logger.NewLogfLogger(t)) + pgh := server.NewPostgresHandler(m.API, logger.NewLogfLogger(t), server.SqlV1) m.MustCreateIndex(t, "i", pilosa.IndexOptions{TrackExistence: true}) m.MustCreateField(t, "i", "set") diff --git a/server/server.go b/server/server.go index 5a8478513..8d227ade7 100644 --- a/server/server.go +++ b/server/server.go @@ -40,7 +40,7 @@ import ( "golang.org/x/sync/errgroup" - "github.com/molecula/featurebase/v2" + pilosa "github.com/molecula/featurebase/v2" "github.com/molecula/featurebase/v2/boltdb" "github.com/molecula/featurebase/v2/encoding/proto" petcd "github.com/molecula/featurebase/v2/etcd" @@ -265,7 +265,7 @@ func (m *Command) Start() (err error) { } tlsConf = conf } - m.pgserver = NewPostgresServer(m.API, m.logger, tlsConf) + m.pgserver = NewPostgresServer(m.API, m.logger, tlsConf, SqlVersion(m.Config.Postgres.SqlVersion)) m.pgserver.s.StartupTimeout = time.Duration(m.Config.Postgres.StartupTimeout) m.pgserver.s.ReadTimeout = time.Duration(m.Config.Postgres.ReadTimeout) m.pgserver.s.WriteTimeout = time.Duration(m.Config.Postgres.WriteTimeout) diff --git a/server_internal_test.go b/server_internal_test.go index 7fbddf73e..7a1b7b121 100644 --- a/server_internal_test.go +++ b/server_internal_test.go @@ -19,6 +19,7 @@ import ( "testing" "time" + "github.com/molecula/featurebase/v2/storage" "github.com/molecula/featurebase/v2/testhook" ) @@ -45,8 +46,9 @@ func TestMonitorAntiEntropyZero(t *testing.T) { if err != nil { t.Fatalf("getting temp dir: %v", err) } + cfg := &storage.Config{FsyncEnabled: false, Backend: storage.DefaultBackend} s, err := NewServer(OptServerDataDir(td), - OptServerAntiEntropyInterval(0)) + OptServerAntiEntropyInterval(0), OptServerStorageConfig(cfg)) if err != nil { t.Fatalf("making new server: %v", err) } diff --git a/sql/mapper.go b/sql/mapper.go index 3ff500d8f..80ce64b6c 100644 --- a/sql/mapper.go +++ b/sql/mapper.go @@ -25,6 +25,8 @@ import ( const ( SQLTypeSelect = "select" SQLTypeShow = "show" + SQLTypeSet = "set" + SQLTypeBegin = "begin" SQLTypeEmpty = "" ) @@ -101,6 +103,10 @@ func (m *Mapper) MapSQL(sql string) (*MappedSQL, error) { } case *sqlparser.Show: sqlType = SQLTypeShow + case *sqlparser.Set: + sqlType = SQLTypeSet + case *sqlparser.Begin: + sqlType = SQLTypeBegin } return &MappedSQL{ diff --git a/sql2/ast.go b/sql2/ast.go index ce231781f..9d2e1a4db 100644 --- a/sql2/ast.go +++ b/sql2/ast.go @@ -194,6 +194,26 @@ func StatementSource(stmt Statement) Source { } } +// Data types +const ( + DataTypeBool = "BOOL" + DataTypeDecimal = "DECIMAL" + DataTypeInt = "INT" + DataTypeSet = "SET" + DataTypeText = "TEXT" + DataTypeTimestamp = "TIMESTAMP" +) + +// IsDataTypeValid returns true if typ is a valid data type. +func IsDataTypeValid(typ string) bool { + switch typ { + case DataTypeBool, DataTypeInt, DataTypeDecimal, DataTypeText: + return true + default: + return false + } +} + type Expr interface { Node expr() @@ -279,6 +299,49 @@ func cloneExprs(a []Expr) []Expr { return other } +// ExprDataType returns the data type for an expression. +func ExprDataType(expr Expr) string { + if expr == nil { + return "" + } + + switch expr := expr.(type) { + // Simple type assertions + case *BindExpr, *ExprList, *Ident, *NullLit, *Raise: + return "" + case *BlobLit, *StringLit: + return DataTypeText + case *BoolLit, *Exists, *Range: + return DataTypeBool + case *NumberLit: + return DataTypeInt + + // Complex type assertions + case *BinaryExpr: + return ExprDataType(expr.X) + case *Call: + return DataTypeInt // TODO: May be different for some aggregations + case *CaseExpr: + if len(expr.Blocks) > 0 { + return ExprDataType(expr.Blocks[0].Body) + } else if expr.ElseExpr != nil { + return ExprDataType(expr.ElseExpr) + } + return "" + case *CastExpr: + return "" // TODO: Inspect expr.Type.Name + case *ParenExpr: + return ExprDataType(expr.X) + case *QualifiedRef: + return expr.DataType + case *UnaryExpr: + return ExprDataType(expr.X) + + default: + panic(fmt.Sprintf("invalid expr type: %T", expr)) + } +} + // ExprString returns the string representation of expr. // Returns a blank string if expr is nil. func ExprString(expr Expr) string { @@ -288,6 +351,119 @@ func ExprString(expr Expr) string { return expr.String() } +// ExprTableName returns the name of the table referenced in an expression. +// Returns ok as false if more than one table referenced. Returns a blank string +// if no tables are referenced. +func ExprTableName(expr Expr) (table string, ok bool) { + switch expr := expr.(type) { + case *BindExpr, *BlobLit, *BoolLit, *Ident, *NullLit, *NumberLit, *StringLit: + return "", true + + case *BinaryExpr: + x, ok := ExprTableName(expr.X) + if !ok { + return "", false + } + + y, ok := ExprTableName(expr.Y) + if !ok { + return "", false + } + + if x == "" { + return y, true + } else if y == "" { + return x, true + } else if x == y { + return x, true + } + return "", false + + case *Call: + for _, arg := range expr.Args { + tbl, ok := ExprTableName(arg) + if !ok || (table != "" && tbl != table) { + return "", false + } + table = tbl + } + return table, true + + case *CaseExpr: + tbl, ok := ExprTableName(expr.Operand) + if !ok || (table != "" && tbl != table) { + return "", false + } + table = tbl + + tbl, ok = ExprTableName(expr.ElseExpr) + if !ok || (table != "" && tbl != table) { + return "", false + } + table = tbl + + for _, blk := range expr.Blocks { + tbl, ok := ExprTableName(blk.Condition) + if !ok || (table != "" && tbl != table) { + return "", false + } + table = tbl + + tbl, ok = ExprTableName(blk.Body) + if !ok || (table != "" && tbl != table) { + return "", false + } + table = tbl + } + return table, true + + case *CastExpr: + return ExprTableName(expr.X) + + case *Exists: + return "", false // TODO + + case *ExprList: + for _, e := range expr.Exprs { + tbl, ok := ExprTableName(e) + if !ok || (table != "" && tbl != table) { + return "", false + } + table = tbl + } + return table, true + + case *ParenExpr: + return ExprTableName(expr.X) + + case *QualifiedRef: + return expr.Table.Name, true + + case *Raise: + return "", true + + case *Range: + tbl, ok := ExprTableName(expr.X) + if !ok || (table != "" && tbl != table) { + return "", false + } + table = tbl + + tbl, ok = ExprTableName(expr.Y) + if !ok || (table != "" && tbl != table) { + return "", false + } + table = tbl + return table, true + + case *UnaryExpr: + return ExprTableName(expr.X) + + default: + return "", false + } +} + // SplitExprTree splits apart expr so it is a list of all AND joined expressions. // For example, the expression "A AND B AND (C OR (D AND E))" would be split into // a list of "A", "B", "C OR (D AND E)". @@ -1811,6 +1987,9 @@ type QualifiedRef struct { Dot Pos // position of dot Star Pos // position of * (result column only) Column *Ident // column name + + // Set by the planner; not at parse-time + DataType string } // IsAggregate returns false. @@ -2935,6 +3114,23 @@ func (s *SelectStatement) IsAggregate() bool { return false } +// HasWildcard returns true any result column contains a wildcard (STAR). +func (s *SelectStatement) HasWildcard() bool { + for _, col := range s.Columns { + // Unqualified wildcard. + if col.Star.IsValid() { + return true + } + + // Table-qualified wildcard. + if ref, ok := col.Expr.(*QualifiedRef); ok && ref.Star.IsValid() { + return true + } + } + + return false +} + // String returns the string representation of the statement. func (s *SelectStatement) String() string { var buf bytes.Buffer @@ -3061,10 +3257,12 @@ func (c *ResultColumn) Name() string { } switch expr := c.Expr.(type) { + case *Call: + return strings.ToLower(IdentName(expr.Name)) case *Ident: return IdentName(expr) case *QualifiedRef: - return expr.String() + return IdentName(expr.Column) default: return "" } diff --git a/sql2/ast_test.go b/sql2/ast_test.go index 8d3c1fed5..25cf95ac9 100644 --- a/sql2/ast_test.go +++ b/sql2/ast_test.go @@ -1143,14 +1143,14 @@ func AssertNodeStringerPanic(tb testing.TB, node sql.Node, msg string) { func StripPos(root sql.Node) sql.Node { zero := reflect.ValueOf(sql.Pos{}) - _ = sql.Walk(sql.VisitFunc(func(node sql.Node) error { + _, _ = sql.Walk(sql.VisitFunc(func(node sql.Node) (sql.Node, error) { value := reflect.Indirect(reflect.ValueOf(node)) for i := 0; i < value.NumField(); i++ { if field := value.Field(i); field.Type() == zero.Type() { field.Set(zero) } } - return nil + return node, nil }), root) return root } diff --git a/sql2/scanner.go b/sql2/scanner.go index 552034a83..105f321b1 100644 --- a/sql2/scanner.go +++ b/sql2/scanner.go @@ -104,6 +104,11 @@ func (s *Scanner) Scan() (pos Pos, token Token, lit string) { case '+': return pos, PLUS, "+" case '-': + if s.peek() == '-' { + s.read() + s.skipComment() + continue + } return pos, MINUS, "-" case '*': return pos, STAR, "*" @@ -117,6 +122,13 @@ func (s *Scanner) Scan() (pos Pos, token Token, lit string) { } } +// skipComment reads all characters until the end of the line or EOF. +func (s *Scanner) skipComment() { + for ch := s.peek(); ch != '\n' && ch != -1; ch = s.peek() { + s.read() + } +} + func (s *Scanner) scanUnquotedIdent(pos Pos, prefix string) (Pos, Token, string) { assert(isUnquotedIdent(s.peek())) diff --git a/sql2/scanner_test.go b/sql2/scanner_test.go index 27e33222d..f4a25a0aa 100644 --- a/sql2/scanner_test.go +++ b/sql2/scanner_test.go @@ -38,6 +38,9 @@ func TestScanner_Scan(t *testing.T) { t.Run("StartingX", func(t *testing.T) { AssertScan(t, `xyz`, sql.IDENT, `xyz`) }) + t.Run("WithComment", func(t *testing.T) { + AssertScan(t, "-- this is a comment\n\n-- more comments\nfoo", sql.IDENT, `foo`) + }) }) t.Run("KEYWORD", func(t *testing.T) { diff --git a/sql2/walk.go b/sql2/walk.go index e8592a7b6..c04ca76ba 100644 --- a/sql2/walk.go +++ b/sql2/walk.go @@ -18,8 +18,8 @@ package sql2 // If the result visitor w is not nil, Walk visits each of the children // of node with the visitor w, followed by a call of w.Visit(nil). type Visitor interface { - Visit(node Node) (w Visitor, err error) - VisitEnd(node Node) error + Visit(node Node) (w Visitor, n Node, err error) + VisitEnd(node Node) (Node, error) } // Walk traverses an AST in depth-first order: It starts by calling @@ -27,606 +27,774 @@ type Visitor interface { // v.Visit(node) is not nil, Walk is invoked recursively with visitor // w for each of the non-nil children of node, followed by a call of // w.Visit(nil). -func Walk(v Visitor, node Node) error { +func Walk(v Visitor, node Node) (Node, error) { return walk(v, node) } -func walk(v Visitor, node Node) (err error) { +func walk(v Visitor, node Node) (_ Node, err error) { // Visit the node itself - if v, err = v.Visit(node); err != nil { - return err + if v, node, err = v.Visit(node); err != nil { + return node, err } else if v == nil { - return nil + return node, nil } // Visit node's children. switch n := node.(type) { case *Assignment: if err := walkIdentList(v, n.Columns); err != nil { - return err + return node, err } - if err := walkExpr(v, n.Expr); err != nil { - return err + if err := walkExpr(v, &n.Expr); err != nil { + return node, err } case *ExplainStatement: if n.Stmt != nil { - if err := walk(v, n.Stmt); err != nil { - return err + if stmt, err := walk(v, n.Stmt); err != nil { + return node, err + } else if stmt == nil { + n.Stmt = nil + } else { + n.Stmt = stmt.(Statement) } } case *RollbackStatement: - if err := walkIdent(v, n.SavepointName); err != nil { - return err + if err := walkIdent(v, &n.SavepointName); err != nil { + return node, err } case *SavepointStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } case *ReleaseStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } case *CreateTableStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if err := walkColumnDefinitionList(v, n.Columns); err != nil { - return err + return node, err } if err := walkConstraintList(v, n.Constraints); err != nil { - return err + return node, err } if n.Select != nil { - if err := walk(v, n.Select); err != nil { - return err + if sel, err := walk(v, n.Select); err != nil { + return node, err + } else if sel != nil { + n.Select = sel.(*SelectStatement) + } else { + n.Select = nil } } case *AlterTableStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } - if err := walkIdent(v, n.NewName); err != nil { - return err + if err := walkIdent(v, &n.NewName); err != nil { + return node, err } - if err := walkIdent(v, n.ColumnName); err != nil { - return err + if err := walkIdent(v, &n.ColumnName); err != nil { + return node, err } - if err := walkIdent(v, n.NewColumnName); err != nil { - return err + if err := walkIdent(v, &n.NewColumnName); err != nil { + return node, err } if n.ColumnDef != nil { - if err := walk(v, n.ColumnDef); err != nil { - return err + if def, err := walk(v, n.ColumnDef); err != nil { + return node, err + } else if def != nil { + n.ColumnDef = def.(*ColumnDefinition) + } else { + n.ColumnDef = nil } } case *AnalyzeStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } case *CreateViewStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if err := walkIdentList(v, n.Columns); err != nil { - return err + return node, err } if n.Select != nil { - if err := walk(v, n.Select); err != nil { - return err + if sel, err := walk(v, n.Select); err != nil { + return node, err + } else if sel != nil { + n.Select = sel.(*SelectStatement) + } else { + n.Select = nil } } case *DropTableStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } case *DropViewStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } case *DropIndexStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } case *DropTriggerStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } case *CreateIndexStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } - if err := walkIdent(v, n.Table); err != nil { - return err + if err := walkIdent(v, &n.Table); err != nil { + return node, err } if err := walkIndexedColumnList(v, n.Columns); err != nil { - return err + return node, err } - if err := walkExpr(v, n.WhereExpr); err != nil { - return err + if err := walkExpr(v, &n.WhereExpr); err != nil { + return node, err } case *CreateTriggerStatement: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if err := walkIdentList(v, n.UpdateOfColumns); err != nil { - return err + return node, err } - if err := walkIdent(v, n.Table); err != nil { - return err + if err := walkIdent(v, &n.Table); err != nil { + return node, err } - if err := walkExpr(v, n.WhenExpr); err != nil { - return err + if err := walkExpr(v, &n.WhenExpr); err != nil { + return node, err } - for _, x := range n.Body { - if err := walk(v, x); err != nil { - return err + for i := range n.Body { + if body, err := walk(v, n.Body[i]); err != nil { + return node, err + } else if body != nil { + n.Body[i] = body.(Statement) + } else { + n.Body[i] = nil } } case *SelectStatement: if n.WithClause != nil { - if err := walk(v, n.WithClause); err != nil { - return err + if clause, err := walk(v, n.WithClause); err != nil { + return node, err + } else if clause != nil { + n.WithClause = clause.(*WithClause) + } else { + n.WithClause = nil } } - for _, x := range n.ValueLists { - if err := walk(v, x); err != nil { - return err + for i := range n.ValueLists { + if list, err := walk(v, n.ValueLists[i]); err != nil { + return node, err + } else if list != nil { + n.ValueLists[i] = list.(*ExprList) + } else { + n.ValueLists[i] = nil } } - for _, x := range n.Columns { - if err := walk(v, x); err != nil { - return err + for i := range n.Columns { + if col, err := walk(v, n.Columns[i]); err != nil { + return node, err + } else if col != nil { + n.Columns[i] = col.(*ResultColumn) + } else { + n.Columns[i] = nil } } if n.Source != nil { - if err := walk(v, n.Source); err != nil { - return err + if src, err := walk(v, n.Source); err != nil { + return node, err + } else if src != nil { + n.Source = n.Source.(Source) + } else { + n.Source = nil } } - if err := walkExpr(v, n.WhereExpr); err != nil { - return err + if err := walkExpr(v, &n.WhereExpr); err != nil { + return node, err } if err := walkExprList(v, n.GroupByExprs); err != nil { - return err + return node, err } - if err := walkExpr(v, n.HavingExpr); err != nil { - return err + if err := walkExpr(v, &n.HavingExpr); err != nil { + return node, err } - for _, x := range n.Windows { - if err := walk(v, x); err != nil { - return err + for i := range n.Windows { + if w, err := walk(v, n.Windows[i]); err != nil { + return node, err + } else if w != nil { + n.Windows[i] = w.(*Window) + } else { + n.Windows[i] = nil } } if n.Compound != nil { - if err := walk(v, n.Compound); err != nil { - return err + if stmt, err := walk(v, n.Compound); err != nil { + return node, err + } else if stmt != nil { + n.Compound = stmt.(*SelectStatement) + } else { + n.Compound = nil } } - for _, x := range n.OrderingTerms { - if err := walk(v, x); err != nil { - return err + for i := range n.OrderingTerms { + if term, err := walk(v, n.OrderingTerms[i]); err != nil { + return node, err + } else if term != nil { + n.OrderingTerms[i] = term.(*OrderingTerm) + } else { + n.OrderingTerms[i] = nil } } - if err := walkExpr(v, n.LimitExpr); err != nil { - return err + if err := walkExpr(v, &n.LimitExpr); err != nil { + return node, err } - if err := walkExpr(v, n.OffsetExpr); err != nil { - return err + if err := walkExpr(v, &n.OffsetExpr); err != nil { + return node, err } case *InsertStatement: if n.WithClause != nil { - if err := walk(v, n.WithClause); err != nil { - return err + if clause, err := walk(v, n.WithClause); err != nil { + return node, err + } else if clause != nil { + n.WithClause = clause.(*WithClause) + } else { + n.WithClause = nil } } - if err := walkIdent(v, n.Table); err != nil { - return err + if err := walkIdent(v, &n.Table); err != nil { + return node, err } - if err := walkIdent(v, n.Alias); err != nil { - return err + if err := walkIdent(v, &n.Alias); err != nil { + return node, err } if err := walkIdentList(v, n.Columns); err != nil { - return err + return node, err } - for _, x := range n.ValueLists { - if err := walk(v, x); err != nil { - return err + for i := range n.ValueLists { + if list, err := walk(v, n.ValueLists[i]); err != nil { + return node, err + } else if list != nil { + n.ValueLists[i] = list.(*ExprList) + } else { + n.ValueLists[i] = nil } } if n.Select != nil { - if err := walk(v, n.Select); err != nil { - return err + if sel, err := walk(v, n.Select); err != nil { + return node, err + } else if sel != nil { + n.Select = sel.(*SelectStatement) + } else { + n.Select = nil } } if n.UpsertClause != nil { - if err := walk(v, n.UpsertClause); err != nil { - return err + if clause, err := walk(v, n.UpsertClause); err != nil { + return node, err + } else if clause != nil { + n.UpsertClause = clause.(*UpsertClause) + } else { + n.UpsertClause = nil } } case *UpdateStatement: if n.WithClause != nil { - if err := walk(v, n.WithClause); err != nil { - return err + if clause, err := walk(v, n.WithClause); err != nil { + return node, err + } else if clause != nil { + n.WithClause = clause.(*WithClause) + } else { + n.WithClause = nil } } if n.Table != nil { - if err := walk(v, n.Table); err != nil { - return err + if tbl, err := walk(v, n.Table); err != nil { + return node, err + } else if tbl != nil { + n.Table = tbl.(*QualifiedTableName) + } else { + n.Table = nil } } - for _, x := range n.Assignments { - if err := walk(v, x); err != nil { - return err + for i := range n.Assignments { + if assign, err := walk(v, n.Assignments[i]); err != nil { + return node, err + } else if assign != nil { + n.Assignments[i] = assign.(*Assignment) + } else { + n.Assignments[i] = nil } } - if err := walkExpr(v, n.WhereExpr); err != nil { - return err + if err := walkExpr(v, &n.WhereExpr); err != nil { + return node, err } case *UpsertClause: if err := walkIndexedColumnList(v, n.Columns); err != nil { - return err + return node, err } - if err := walkExpr(v, n.WhereExpr); err != nil { - return err + if err := walkExpr(v, &n.WhereExpr); err != nil { + return node, err } - for _, x := range n.Assignments { - if err := walk(v, x); err != nil { - return err + for i := range n.Assignments { + if assign, err := walk(v, n.Assignments[i]); err != nil { + return node, err + } else if assign != nil { + n.Assignments[i] = assign.(*Assignment) + } else { + n.Assignments[i] = nil } } - if err := walkExpr(v, n.UpdateWhereExpr); err != nil { - return err + if err := walkExpr(v, &n.UpdateWhereExpr); err != nil { + return node, err } case *DeleteStatement: if n.WithClause != nil { - if err := walk(v, n.WithClause); err != nil { - return err + if clause, err := walk(v, n.WithClause); err != nil { + return node, err + } else if clause != nil { + n.WithClause = clause.(*WithClause) + } else { + n.WithClause = nil } } if n.Table != nil { - if err := walk(v, n.Table); err != nil { - return err + if tbl, err := walk(v, n.Table); err != nil { + return node, err + } else if tbl != nil { + n.Table = tbl.(*QualifiedTableName) + } else { + n.Table = nil } } - if err := walkExpr(v, n.WhereExpr); err != nil { - return err + if err := walkExpr(v, &n.WhereExpr); err != nil { + return node, err } - for _, x := range n.OrderingTerms { - if err := walk(v, x); err != nil { - return err + for i := range n.OrderingTerms { + if term, err := walk(v, n.OrderingTerms[i]); err != nil { + return node, err + } else if term != nil { + n.OrderingTerms[i] = term.(*OrderingTerm) + } else { + n.OrderingTerms[i] = nil } } - if err := walkExpr(v, n.LimitExpr); err != nil { - return err + if err := walkExpr(v, &n.LimitExpr); err != nil { + return node, err } - if err := walkExpr(v, n.OffsetExpr); err != nil { - return err + if err := walkExpr(v, &n.OffsetExpr); err != nil { + return node, err } case *PrimaryKeyConstraint: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if err := walkIdentList(v, n.Columns); err != nil { - return err + return node, err } case *NotNullConstraint: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } case *UniqueConstraint: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if err := walkIdentList(v, n.Columns); err != nil { - return err + return node, err } case *CheckConstraint: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } - if err := walkExpr(v, n.Expr); err != nil { - return err + if err := walkExpr(v, &n.Expr); err != nil { + return node, err } case *DefaultConstraint: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } - if err := walkExpr(v, n.Expr); err != nil { - return err + if err := walkExpr(v, &n.Expr); err != nil { + return node, err } case *ForeignKeyConstraint: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if err := walkIdentList(v, n.Columns); err != nil { - return err + return node, err } - if err := walkIdent(v, n.ForeignTable); err != nil { - return err + if err := walkIdent(v, &n.ForeignTable); err != nil { + return node, err } if err := walkIdentList(v, n.ForeignColumns); err != nil { - return err + return node, err } - for _, x := range n.Args { - if err := walk(v, x); err != nil { - return err + for i := range n.Args { + if arg, err := walk(v, n.Args[i]); err != nil { + return node, err + } else if arg != nil { + n.Args[i] = arg.(*ForeignKeyArg) + } else { + n.Args[i] = nil } } case *ParenExpr: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } case *UnaryExpr: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } case *BinaryExpr: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } - if err := walkExpr(v, n.Y); err != nil { - return err + if err := walkExpr(v, &n.Y); err != nil { + return node, err } case *CastExpr: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } if n.Type != nil { - if err := walk(v, n.Type); err != nil { - return err + if typ, err := walk(v, n.Type); err != nil { + return node, err + } else if typ != nil { + n.Type = typ.(*Type) + } else { + n.Type = nil } } case *CaseBlock: - if err := walkExpr(v, n.Condition); err != nil { - return err + if err := walkExpr(v, &n.Condition); err != nil { + return node, err } - if err := walkExpr(v, n.Body); err != nil { - return err + if err := walkExpr(v, &n.Body); err != nil { + return node, err } case *CaseExpr: - if err := walkExpr(v, n.Operand); err != nil { - return err + if err := walkExpr(v, &n.Operand); err != nil { + return node, err } - for _, x := range n.Blocks { - if err := walk(v, x); err != nil { - return err + for i := range n.Blocks { + if blk, err := walk(v, n.Blocks[i]); err != nil { + return node, err + } else if blk != nil { + n.Blocks[i] = blk.(*CaseBlock) + } else { + n.Blocks[i] = nil } } - if err := walkExpr(v, n.ElseExpr); err != nil { - return err + if err := walkExpr(v, &n.ElseExpr); err != nil { + return node, err } case *ExprList: if err := walkExprList(v, n.Exprs); err != nil { - return err + return node, err } case *QualifiedRef: - if err := walkIdent(v, n.Table); err != nil { - return err + if err := walkIdent(v, &n.Table); err != nil { + return node, err } - if err := walkIdent(v, n.Column); err != nil { - return err + if err := walkIdent(v, &n.Column); err != nil { + return node, err } case *Call: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if err := walkExprList(v, n.Args); err != nil { - return err + return node, err } if n.Filter != nil { - if err := walk(v, n.Filter); err != nil { - return err + if filter, err := walk(v, n.Filter); err != nil { + return node, err + } else if filter != nil { + n.Filter = filter.(*FilterClause) + } else { + n.Filter = nil } } if n.Over != nil { - if err := walk(v, n.Over); err != nil { - return err + if over, err := walk(v, n.Over); err != nil { + return node, err + } else if over != nil { + n.Over = over.(*OverClause) + } else { + n.Over = nil } } case *FilterClause: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } case *OverClause: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if n.Definition != nil { - if err := walk(v, n.Definition); err != nil { - return err + if def, err := walk(v, n.Definition); err != nil { + return node, err + } else if def != nil { + n.Definition = def.(*WindowDefinition) + } else { + n.Definition = nil } } case *OrderingTerm: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } case *FrameSpec: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } - if err := walkExpr(v, n.Y); err != nil { - return err + if err := walkExpr(v, &n.Y); err != nil { + return node, err } case *Range: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } - if err := walkExpr(v, n.Y); err != nil { - return err + if err := walkExpr(v, &n.Y); err != nil { + return node, err } case *Raise: if n.Error != nil { - if err := walk(v, n.Error); err != nil { - return err + if e, err := walk(v, n.Error); err != nil { + return node, err + } else if e != nil { + n.Error = e.(*StringLit) + } else { + n.Error = nil } } case *Exists: if n.Select != nil { - if err := walk(v, n.Select); err != nil { - return err + if sel, err := walk(v, n.Select); err != nil { + return node, err + } else if sel != nil { + n.Select = sel.(*SelectStatement) + } else { + n.Select = nil } } case *ParenSource: if n.X != nil { - if err := walk(v, n.X); err != nil { - return err + if x, err := walk(v, n.X); err != nil { + return node, err + } else if x != nil { + n.X = x.(Source) + } else { + n.X = nil } } - if err := walkIdent(v, n.Alias); err != nil { - return err + if err := walkIdent(v, &n.Alias); err != nil { + return node, err } case *QualifiedTableName: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } - if err := walkIdent(v, n.Alias); err != nil { - return err + if err := walkIdent(v, &n.Alias); err != nil { + return node, err } - if err := walkIdent(v, n.Index); err != nil { - return err + if err := walkIdent(v, &n.Index); err != nil { + return node, err } case *JoinClause: if n.X != nil { - if err := walk(v, n.X); err != nil { - return err + if x, err := walk(v, n.X); err != nil { + return node, err + } else if x != nil { + n.X = x.(Source) + } else { + n.X = nil } } if n.Operator != nil { - if err := walk(v, n.Operator); err != nil { - return err + if op, err := walk(v, n.Operator); err != nil { + return node, err + } else if op != nil { + n.Operator = op.(*JoinOperator) + } else { + n.Operator = nil } } if n.Y != nil { - if err := walk(v, n.Y); err != nil { - return err + if y, err := walk(v, n.Y); err != nil { + return node, err + } else if y != nil { + n.Y = y.(Source) + } else { + n.Y = nil } } if n.Constraint != nil { - if err := walk(v, n.Constraint); err != nil { - return err + if cons, err := walk(v, n.Constraint); err != nil { + return node, err + } else if cons != nil { + n.Constraint = cons.(JoinConstraint) + } else { + n.Constraint = nil } } case *OnConstraint: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } case *UsingConstraint: if err := walkIdentList(v, n.Columns); err != nil { - return err + return node, err } case *ColumnDefinition: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if n.Type != nil { - if err := walk(v, n.Type); err != nil { - return err + if typ, err := walk(v, n.Type); err != nil { + return node, err + } else if typ != nil { + n.Type = typ.(*Type) + } else { + n.Type = nil } } if err := walkConstraintList(v, n.Constraints); err != nil { - return err + return node, err } case *ResultColumn: - if err := walkExpr(v, n.Expr); err != nil { - return err + if err := walkExpr(v, &n.Expr); err != nil { + return node, err } - if err := walkIdent(v, n.Alias); err != nil { - return err + if err := walkIdent(v, &n.Alias); err != nil { + return node, err } case *IndexedColumn: - if err := walkExpr(v, n.X); err != nil { - return err + if err := walkExpr(v, &n.X); err != nil { + return node, err } case *Window: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if n.Definition != nil { - if err := walk(v, n.Definition); err != nil { - return err + if def, err := walk(v, n.Definition); err != nil { + return node, err + } else if def != nil { + n.Definition = def.(*WindowDefinition) + } else { + n.Definition = nil } } case *WindowDefinition: - if err := walkIdent(v, n.Base); err != nil { - return err + if err := walkIdent(v, &n.Base); err != nil { + return node, err } if err := walkExprList(v, n.Partitions); err != nil { - return err + return node, err } - for _, x := range n.OrderingTerms { - if err := walk(v, x); err != nil { - return err + for i := range n.OrderingTerms { + if term, err := walk(v, n.OrderingTerms[i]); err != nil { + return node, err + } else if term != nil { + n.OrderingTerms[i] = term.(*OrderingTerm) + } else { + n.OrderingTerms[i] = nil } } if n.Frame != nil { - if err := walk(v, n.Frame); err != nil { - return err + if frame, err := walk(v, n.Frame); err != nil { + return node, err + } else if frame != nil { + n.Frame = frame.(*FrameSpec) + } else { + n.Frame = nil } } case *Type: - if err := walkIdent(v, n.Name); err != nil { - return err + if err := walkIdent(v, &n.Name); err != nil { + return node, err } if n.Precision != nil { - if err := walk(v, n.Precision); err != nil { - return err + if p, err := walk(v, n.Precision); err != nil { + return node, err + } else if p != nil { + n.Precision = p.(*NumberLit) + } else { + n.Precision = nil } } if n.Scale != nil { - if err := walk(v, n.Scale); err != nil { - return err + if scale, err := walk(v, n.Scale); err != nil { + return node, err + } else if scale != nil { + n.Scale = scale.(*NumberLit) + } else { + n.Scale = nil } } } @@ -637,59 +805,72 @@ func walk(v Visitor, node Node) (err error) { // VisitFunc represents a function type that implements Visitor. // Only executes on node entry. -type VisitFunc func(Node) error +type VisitFunc func(Node) (Node, error) // Visit executes fn. Walk visits node children if fn returns true. -func (fn VisitFunc) Visit(node Node) (Visitor, error) { - if err := fn(node); err != nil { - return nil, err +func (fn VisitFunc) Visit(node Node) (Visitor, Node, error) { + node, err := fn(node) + if err != nil { + return nil, nil, err } - return fn, nil + return fn, node, nil } // VisitEnd is a no-op. -func (fn VisitFunc) VisitEnd(node Node) error { return nil } +func (fn VisitFunc) VisitEnd(node Node) (Node, error) { return node, nil } // VisitEndFunc represents a function type that implements Visitor. // Only executes on node exit. -type VisitEndFunc func(Node) error +type VisitEndFunc func(Node) (Node, error) // Visit is a no-op. -func (fn VisitEndFunc) Visit(node Node) (Visitor, error) { return fn, nil } +func (fn VisitEndFunc) Visit(node Node) (Visitor, Node, error) { return fn, node, nil } // VisitEnd executes fn. -func (fn VisitEndFunc) VisitEnd(node Node) error { return fn(node) } +func (fn VisitEndFunc) VisitEnd(node Node) (Node, error) { return fn(node) } -func walkIdent(v Visitor, x *Ident) error { - if x != nil { - if err := walk(v, x); err != nil { - return err - } +func walkIdent(v Visitor, x **Ident) error { + if *x == nil { + return nil + } + + ident, err := walk(v, *x) + if err != nil { + return err + } else if ident != nil { + *x = ident.(*Ident) + } else { + *x = nil } return nil } func walkIdentList(v Visitor, a []*Ident) error { - for _, x := range a { - if err := walk(v, x); err != nil { + for i := range a { + if err := walkIdent(v, &a[i]); err != nil { return err } } return nil } -func walkExpr(v Visitor, x Expr) error { - if x != nil { - if err := walk(v, x); err != nil { - return err - } +func walkExpr(v Visitor, x *Expr) error { + if *x == nil { + return nil + } + if other, err := walk(v, *x); err != nil { + return err + } else if other != nil { + *x = other.(Expr) + } else { + *x = nil } return nil } func walkExprList(v Visitor, a []Expr) error { - for _, x := range a { - if err := walk(v, x); err != nil { + for i := range a { + if err := walkExpr(v, &a[i]); err != nil { return err } } @@ -697,27 +878,39 @@ func walkExprList(v Visitor, a []Expr) error { } func walkConstraintList(v Visitor, a []Constraint) error { - for _, x := range a { - if err := walk(v, x); err != nil { + for i := range a { + if cons, err := walk(v, a[i]); err != nil { return err + } else if cons != nil { + a[i] = cons.(Constraint) + } else { + a[i] = nil } } return nil } func walkIndexedColumnList(v Visitor, a []*IndexedColumn) error { - for _, x := range a { - if err := walk(v, x); err != nil { + for i := range a { + if col, err := walk(v, a[i]); err != nil { return err + } else if col != nil { + a[i] = col.(*IndexedColumn) + } else { + a[i] = nil } } return nil } func walkColumnDefinitionList(v Visitor, a []*ColumnDefinition) error { - for _, x := range a { - if err := walk(v, x); err != nil { + for i := range a { + if def, err := walk(v, a[i]); err != nil { return err + } else if def != nil { + a[i] = def.(*ColumnDefinition) + } else { + a[i] = nil } } return nil diff --git a/syswrap/mmap.go b/syswrap/mmap.go index 238f6dd6f..2d89ce39b 100644 --- a/syswrap/mmap.go +++ b/syswrap/mmap.go @@ -20,6 +20,7 @@ import ( "sync" "sync/atomic" "syscall" + "strings" "github.com/pkg/errors" ) @@ -56,6 +57,9 @@ func Mmap(fd int, offset int64, length int, prot int, flags int) (data []byte, e data, err = syscall.Mmap(fd, offset, length, prot, flags) if err != nil { atomic.AddUint64(&mapCount, ^uint64(0)) // decrement + if strings.Contains(err.Error(), "cannot allocate memory") { + err = errors.New("mmap 'cannot allocate memory' — please see the troubleshooting how-to in the FeatureBase docs.") + } } return data, err } diff --git a/test/cluster.go b/test/cluster.go index 126245a46..137aa6c83 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -607,7 +607,7 @@ func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption { pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond), pilosa.OptServerStorageConfig(&storage.Config{ Backend: pilosa.CurrentBackendOrDefault(), - FsyncEnabled: true, + FsyncEnabled: false, }), ), } diff --git a/test/disco.go b/test/disco.go index 7aebd8f89..d00c06a31 100644 --- a/test/disco.go +++ b/test/disco.go @@ -111,6 +111,7 @@ func GetPortsGenConfigs(tb testing.TB, nodes []*Command) error { LPeerSocket: []*net.TCPListener{peerListener}, LClientSocket: []*net.TCPListener{clientListener}, BootstrapTimeout: 50 * time.Millisecond, + UnsafeNoFsync: true, } peerUrls[i] = fmt.Sprintf("%s=%s", name, peerURL) } @@ -144,14 +145,16 @@ func GenPortsConfig(tb testing.TB, ports []Ports) []*server.Config { BindGRPC: fmt.Sprintf(":%d", ports[i].Grpc), GRPCListener: ports[i].LsnG, Etcd: etcd.Options{ - Dir: discoDir, - LClientURL: lClientURL, - AClientURL: lClientURL, - LPeerURL: lPeerURL, - APeerURL: lPeerURL, - HeartbeatTTL: 5, - LPeerSocket: []*net.TCPListener{lsnP}, - LClientSocket: []*net.TCPListener{lsnC}, + Dir: discoDir, + LClientURL: lClientURL, + AClientURL: lClientURL, + LPeerURL: lPeerURL, + APeerURL: lPeerURL, + HeartbeatTTL: 5, + LPeerSocket: []*net.TCPListener{lsnP}, + LClientSocket: []*net.TCPListener{lsnC}, + BootstrapTimeout: 50 * time.Millisecond, + UnsafeNoFsync: true, }, } cfgs[i].Cluster.Name = clusterName diff --git a/test/holder.go b/test/holder.go index 079495488..6eb63efac 100644 --- a/test/holder.go +++ b/test/holder.go @@ -38,7 +38,10 @@ func NewHolder(tb testing.TB) *Holder { panic(err) } - h := &Holder{Holder: pilosa.NewHolder(path, nil)} + cfg := pilosa.DefaultHolderConfig() + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false + h := &Holder{Holder: pilosa.NewHolder(path, cfg)} return h } diff --git a/test/index.go b/test/index.go index 608bcae1c..46bc7399b 100644 --- a/test/index.go +++ b/test/index.go @@ -33,7 +33,10 @@ func newIndex(tb testing.TB) *Index { if err != nil { panic(err) } - h := pilosa.NewHolder(path, pilosa.DefaultHolderConfig()) + cfg := pilosa.DefaultHolderConfig() + cfg.StorageConfig.FsyncEnabled = false + cfg.RBFConfig.FsyncEnabled = false + h := pilosa.NewHolder(path, cfg) testhook.Cleanup(tb, func() { h.Close() }) diff --git a/testBackupRestore.sh b/testBackupRestore.sh index 8f9617e7f..7bcbe3894 100755 --- a/testBackupRestore.sh +++ b/testBackupRestore.sh @@ -13,19 +13,19 @@ STATUS=$STATUS timeout -s TERM $TIMEOUT bash -c \ done;' echo "NOW DO STUFF" datagen --source kitchensink_keyed -e 9999 --pilosa.index sink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101 -before=$(/pilosa chksum --host pilosa0:10101) -/pilosa backup -o backupdir --host pilosa0:10101 +before=$(/featurebase chksum --host pilosa0:10101) +/featurebase backup -o backupdir --host pilosa0:10101 curl -X DELETE -s pilosa0:10101/index/sink -/pilosa restore -s backupdir --host pilosa0:10101 -after=$(/pilosa chksum --host pilosa0:10101) +/featurebase restore -s backupdir --host pilosa0:10101 +after=$(/featurebase chksum --host pilosa0:10101) if [ "$before" = "$after" ]; then echo "PASS Cluster" else echo "FAIL Single" exit 1 fi -/pilosa restore -s backupdir --host pilosax:10101 -single=$(/pilosa chksum --host pilosax:10101) +/featurebase restore -s backupdir --host pilosax:10101 +single=$(/featurebase chksum --host pilosax:10101) if [ "$before" = "$single" ]; then echo "PASS Single" exit 0 @@ -33,3 +33,25 @@ else echo "FAIL Single" exit 1 fi + +datagen --source texas_health -e 9999 --pilosa.index newsink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101 +before=$(/featurebase chksum --host pilosa0:10101) +/featurebase backup -o newbackupdir --host pilosa0:10101 --index newsink +curl -X DELETE -s pilosa0:10101/index/newsink +/featurebase restore -s newbackupdir --host pilosa0:10101 +after=$(/featurebase chksum --host pilosa0:10101) +if [ "$before" = "$after" ]; then + echo "PASS Cluster Table" +else + echo "FAIL Single Table" + exit 1 +fi +/featurebase restore -s newbackupdir --host pilosax:10101 +single=$(/featurebase chksum --host pilosax:10101) +if [ "$before" = "$single" ]; then + echo "PASS Single Table" + exit 0 +else + echo "FAIL Single Table" + exit 1 +fi diff --git a/translate.go b/translate.go index 738fc1686..256dae341 100644 --- a/translate.go +++ b/translate.go @@ -23,6 +23,7 @@ import ( "sort" "sync" + "github.com/molecula/featurebase/v2/ingest" "github.com/molecula/featurebase/v2/topology" "github.com/pkg/errors" ) @@ -98,6 +99,39 @@ type TranslateStore interface { // TODO: refactor this interface; readonly shoul ReadFrom(io.Reader) (int64, error) } +// This implements ingest's key translator interface, which differs +// slightly because we want to be able to do fast lookups on arbitrary +// IDs which are not necessarily contiguous small values, so the []string +// from TranslateIDs isn't a good fit. +type ingestKeyTranslator struct { + store TranslateStore +} + +var _ ingest.KeyTranslator = &ingestKeyTranslator{} + +func (i ingestKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) { + return i.store.CreateKeys(keys...) +} + +func (i ingestKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) { + keys, err := i.store.TranslateIDs(ids) + if err != nil { + return nil, err + } + if len(keys) != len(ids) { + return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys)) + } + out := make(map[uint64]string, len(keys)) + for i, id := range ids { + out[id] = keys[i] + } + return out, nil +} + +func newIngestKeyTranslatorFromStore(s TranslateStore) *ingestKeyTranslator { + return &ingestKeyTranslator{store: s} +} + // TranslatorSummary is returned, for example from the boltdb string key translators, // by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method. type TranslatorSummary struct { @@ -163,7 +197,7 @@ TranslatorSummary{ } // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore. -type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int) (TranslateStore, error) +type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (TranslateStore, error) // GenerateNextPartitionedID returns the next ID within the same partition. func GenerateNextPartitionedID(index string, prev uint64, partitionID, partitionN int) uint64 { @@ -373,7 +407,7 @@ var _ OpenTranslateStoreFunc = OpenInMemTranslateStore // OpenInMemTranslateStore returns a new instance of InMemTranslateStore. // Implements OpenTranslateStoreFunc. -func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int) (TranslateStore, error) { +func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int, fsyncEnabled bool) (TranslateStore, error) { return NewInMemTranslateStore(index, field, partitionID, partitionN), nil } diff --git a/utils_internal_test.go b/utils_internal_test.go index a3a6af2a7..76c3aa66f 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -117,7 +117,7 @@ func NewTestClusterWithReplication(tb testing.TB, nNodes, nReplicas, partitionN } // holder - h := NewHolder(path, nil) + h := NewHolder(path, mustHolderConfig()) // cluster availableShardFileFlushDuration.Set(100 * time.Millisecond) diff --git a/view.go b/view.go index bb0eb3a91..ac94f64d8 100644 --- a/view.go +++ b/view.go @@ -300,8 +300,8 @@ func (v *view) Fragment(shard uint64) *fragment { // allFragments returns a list of all fragments in the view. func (v *view) allFragments() []*fragment { - v.mu.Lock() - defer v.mu.Unlock() + v.mu.RLock() + defer v.mu.RUnlock() other := make([]*fragment, 0, len(v.fragments)) for _, fragment := range v.fragments { diff --git a/view_internal_test.go b/view_internal_test.go index 9258cdd27..e7d2036d8 100644 --- a/view_internal_test.go +++ b/view_internal_test.go @@ -35,7 +35,7 @@ func mustOpenView(tb testing.TB, index, field, name string) *view { CacheSize: DefaultCacheSize, } - h := NewHolder(path, nil) + h := NewHolder(path, mustHolderConfig()) // h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment cim := &CreateIndexMessage{