scriptoriumadmin/featurebase
1
0
Fork 0
mirror of https://github.com/featurebasedb/featurebase.git synced 2026-09-11 15:21:02 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into cicd

Browse source
This commit is contained in:
Fletcher Haynes 2021-10-08 12:18:26 -07:00 • committed by GitHub
commit 6eefd2d1f0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
94 changed files with 9423 additions and 1503 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
.circleci/config.yml
View file

@ -35,7 +35,7 @@ commands:
- checkout
- restore-mod-cache
skip-if-root-unchanged:
description: "skips the parent job if the PR includes no changes to pilosa"
description: "skips the parent job if the PR includes no changes to featurebase"
steps:
- run: |
ROOT_CHANGED_FILES="$(git diff --name-only HEAD $(git merge-base master HEAD) | grep -v '^lattice/')" || true
@ -202,7 +202,7 @@ jobs:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker GO_VERSION=1.15.8
- run: docker run pilosa:$(git describe --tags) help
- run: docker run featurebase:$(git describe --tags) help
dockerhub-upload-unstable:
executor:
name: golang
@ -212,8 +212,8 @@ jobs:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker
- run: docker run pilosa:$(git describe --tags) help
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.branch >>
- run: docker run featurebase:$(git describe --tags) help
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.branch >>
dockerhub-upload-stable:
executor:
name: golang
@ -223,9 +223,9 @@ jobs:
version: 19.03.13 # see https://support.circleci.com/hc/en-us/articles/360050934711
- run: echo -n $DOCKER_PASS | docker login -u $DOCKER_USER --password-stdin
- run: make docker
- run: docker run pilosa:$(git describe --tags) help
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:<< pipeline.git.tag >>
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/pilosa:latest
- run: docker run featurebase:$(git describe --tags) help
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:<< pipeline.git.tag >>
- run: make docker-tag-push DOCKER_TARGET=moleculacorp/featurebase:latest
workflows:
build:

6
Dockerfile
View file

@ -28,7 +28,7 @@ COPY . ./
COPY --from=lattice-builder /lattice/build /lattice
RUN /go/bin/statik -src=/lattice -dest=/pilosa
RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
#####################
### Pilosa runner ###
@ -40,7 +40,7 @@ LABEL maintainer "dev@molecula.com"
RUN apk add --no-cache curl jq
COPY --from=pilosa-builder /pilosa/build/pilosa /
COPY --from=pilosa-builder /pilosa/build/featurebase /
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
@ -52,5 +52,5 @@ ENV PILOSA_DATA_DIR /data
ENV PILOSA_BIND 0.0.0.0:10101
ENV PILOSA_BIND_GRPC 0.0.0.0:20101
ENTRYPOINT ["/pilosa"]
ENTRYPOINT ["/featurebase"]
CMD ["server"]

6
Dockerfile-clustertests
View file

@ -1,7 +1,7 @@
# This Dockerfile is used for cluster testing - it produces a much larger image
# and includes all of Go as well as some utilities.
FROM golang:1.13
FROM golang:1.16
LABEL maintainer "dev@pilosa.com"
@ -14,6 +14,10 @@ RUN cd /go/src/github.com/molecula/featurebase \
ADD https://github.com/alexei-led/pumba/releases/download/0.6.0/pumba_linux_amd64 /pumba
RUN chmod +x /pumba
# add docker client to pause/unpause nodes
RUN apt update
RUN apt install -y docker.io
RUN cp /go/bin/featurebase /featurebase
COPY LICENSE /LICENSE

6
Dockerfile.pilosa
View file

@ -10,7 +10,7 @@ WORKDIR /pilosa
COPY . ./
RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
#####################
### Pilosa runner ###
@ -22,7 +22,7 @@ LABEL maintainer "dev@molecula.com"
RUN apk add --no-cache curl jq
COPY --from=pilosa-builder /pilosa/build/pilosa /
COPY --from=pilosa-builder /pilosa/build/featurebase /
COPY LICENSE /LICENSE
COPY NOTICE /NOTICE
@ -34,5 +34,5 @@ ENV PILOSA_DATA_DIR /data
ENV PILOSA_BIND 0.0.0.0:10101
ENV PILOSA_BIND_GRPC 0.0.0.0:20101
ENTRYPOINT ["/pilosa"]
ENTRYPOINT ["/featurebase"]
CMD ["server"]

4
Dockerfile.runner
View file

@ -10,7 +10,7 @@ WORKDIR /pilosa
COPY . ./
RUN make build FLAGS="-o build/pilosa" ${MAKE_FLAGS}
RUN make build FLAGS="-o build/featurebase" ${MAKE_FLAGS}
FROM moleculacorp/idk as idk
LABEL maintainer "dev@molecula.com"
@ -18,6 +18,6 @@ RUN apt-get update -y
RUN apt-get install -y bash curl jq
COPY --from=pilosa-builder /pilosa/build/pilosa /
COPY --from=pilosa-builder /pilosa/build/featurebase /
COPY testBackupRestore.sh /
CMD ["bash","/testBackupRestore.sh"]

26
Makefile
View file

@ -116,6 +116,7 @@ release: check-clean generate-statik-docker
$(MAKE) release-build GOOS=darwin GOARCH=amd64
$(MAKE) release-build GOOS=darwin GOARCH=arm64
$(MAKE) release-build GOOS=linux GOARCH=amd64
$(MAKE) release-build GOOS=linux GOARCH=arm64
# Create release build tarballs for all supported platforms. Same as `release`, but without embedded Lattice UI.
release-sans-ui: check-clean
@ -123,6 +124,7 @@ release-sans-ui: check-clean
$(MAKE) release-build GOOS=darwin GOARCH=amd64
$(MAKE) release-build GOOS=darwin GOARCH=arm64
$(MAKE) release-build GOOS=linux GOARCH=amd64
$(MAKE) release-build GOOS=linux GOARCH=arm64
# try (e.g.) internal/clustertests/docker-compose-replication2.yml
DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
@ -133,10 +135,9 @@ DOCKER_COMPOSE=internal/clustertests/docker-compose.yml
# pilosa image.
clustertests: vendor
docker-compose -f $(DOCKER_COMPOSE) down
docker-compose -f $(DOCKER_COMPOSE) build client1
docker-compose -f $(DOCKER_COMPOSE) build
docker-compose -f $(DOCKER_COMPOSE) up --exit-code-from=client1
# Like clustertests, but rebuilds all images.
clustertests-build: vendor
docker-compose -f $(DOCKER_COMPOSE) down -v
@ -197,6 +198,7 @@ generate: generate-protoc generate-statik generate-stringer generate-pql
# Create release using Docker
docker-release:
$(MAKE) docker-build GOOS=linux GOARCH=amd64
$(MAKE) docker-build GOOS=linux GOARCH=arm64
$(MAKE) docker-build GOOS=darwin GOARCH=amd64
$(MAKE) docker-build GOOS=darwin GOARCH=arm64
@ -206,28 +208,28 @@ docker-build: vendor
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE) GOOS=$(GOOS) GOARCH=$(GOARCH)" \
--target pilosa-builder \
--tag pilosa:build .
docker create --name pilosa-build pilosa:build
mkdir -p build/pilosa-$(VERSION_ID)
docker cp pilosa-build:/pilosa/build/. ./build/pilosa-$(VERSION_ID)
cp NOTICE LICENSE ./build/pilosa-$(VERSION_ID)
docker rm pilosa-build
tar -cvz -C build -f build/pilosa-$(VERSION_ID).tar.gz pilosa-$(VERSION_ID)/
--tag featurebase:build .
docker create --name featurebase-build featurebase:build
mkdir -p build/featurebase-$(VERSION_ID)
docker cp featurebase-build:/pilosa/build/. ./build/featurebase-$(VERSION_ID)
cp NOTICE LICENSE ./build/featurebase-$(VERSION_ID)
docker rm featurebase-build
tar -cvz -C build -f build/featurebase-$(VERSION_ID).tar.gz featurebase-$(VERSION_ID)/
# Create Docker image from Dockerfile
docker-image: vendor
docker build \
--build-arg GO_VERSION=$(GO_VERSION) \
--build-arg MAKE_FLAGS="TRIAL_DEADLINE=$(TRIAL_DEADLINE)" \
--tag pilosa:$(VERSION) .
@echo Created docker image: pilosa:$(VERSION)
--tag featurebase:$(VERSION) .
@echo Created docker image: featurebase:$(VERSION)
# Create docker image (alias)
docker: docker-image # alias
# Tag and push a Docker image
docker-tag-push: vendor
docker tag "pilosa:$(VERSION)" $(DOCKER_TARGET)
docker tag "featurebase:$(VERSION)" $(DOCKER_TARGET)
docker push $(DOCKER_TARGET)
@echo Pushed docker image: $(DOCKER_TARGET)

123
api.go
View file

@ -36,6 +36,8 @@ import (
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/ingest"
//"github.com/molecula/featurebase/v2/pg"
"github.com/molecula/featurebase/v2/pql"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/stats"
@ -1824,6 +1826,44 @@ func (api *API) ImportValueWithTx(ctx context.Context, qcx *Qcx, req *ImportValu
return nil
}
// ingestNodeOperationsForFields does the actual work of applying operations
// to a given index with a map of known fields and an already-parsed
// ShardedRequest. This is used locally on the node that first receives
// the request, after it does the parsing, and on other nodes because the
// format they get is already that rather than JSON, so it's the common
// path *after* key translation and sorting into shards.
func (api *API) ingestNodeOperationsForFields(ctx context.Context, qcx *Qcx, index *Index, knownFields map[string]*Field, req *ingest.ShardedRequest) error {
eg, ctx := errgroup.WithContext(ctx)
for shard, ops := range req.Ops {
// create new local copies of these values so the goroutine uses these
// copies, and doesn't read the actual loop variables, which are being
// changed by the loop.
shard, ops := shard, ops
eg.Go(func() error {
return api.applyOperations(ctx, qcx, index, shard, knownFields, ops)
})
}
return eg.Wait()
}
// IngestNodeOperations handles protobuf-formatted data which does not need
// key translation and is applicable to this specific node.
func (api *API) IngestNodeOperations(ctx context.Context, qcx *Qcx, indexName string, req *ingest.ShardedRequest) error {
index := api.holder.Index(indexName)
if index == nil {
api.server.logger.Errorf("ingest: no such index %q", indexName)
return newNotFoundError(ErrIndexNotFound, indexName)
}
fields := index.Fields()
knownFields := map[string]*Field{}
for _, field := range fields {
knownFields[field.name] = field
}
return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, req)
}
// IngestOperations handles JSON-formatted data which may need key translation
// and may be for any or all nodes.
func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string, stream io.Reader) error {
span, _ := tracing.StartSpanFromContext(ctx, "API.IngestOperations")
defer span.Finish()
@ -1839,34 +1879,32 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string
return newNotFoundError(ErrIndexNotFound, indexName)
}
fields := index.Fields()
var lookup ingest.KeyLookupFunc
var indexKeys ingest.KeyTranslator
if index.Keys() {
lookup = func(keys ...string) (map[string]uint64, error) {
return api.cluster.createIndexKeys(ctx, indexName, keys...)
}
indexKeys = newIngestKeyTranslatorFromCluster(ctx, api.cluster, indexName)
}
codec, err := ingest.NewJSONCodec(lookup)
codec, err := ingest.NewJSONCodec(indexKeys)
if err != nil {
return errors.Wrap(err, "creating JSON codec")
}
knownFields := map[string]*Field{}
for _, field := range fields {
var lookup ingest.KeyLookupFunc
var keys ingest.KeyTranslator
if field.usesKeys {
lookup = field.translateStore.CreateKeys
keys = newIngestKeyTranslatorFromStore(field.translateStore)
}
knownFields[field.name] = field
switch field.Type() {
case "set":
if err = codec.AddSetField(field.name, lookup); err != nil {
if err = codec.AddSetField(field.name, keys); err != nil {
return fmt.Errorf("adding set field to codec: %w", err)
}
case "time":
if err = codec.AddTimeQuantumField(field.name, lookup); err != nil {
if err = codec.AddTimeQuantumField(field.name, keys); err != nil {
return fmt.Errorf("adding time quantum field to codec: %w", err)
}
case "mutex":
if err = codec.AddMutexField(field.name, lookup); err != nil {
if err = codec.AddMutexField(field.name, keys); err != nil {
return fmt.Errorf("adding mutex field to codec: %w", err)
}
case "bool":
@ -1874,7 +1912,7 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string
return fmt.Errorf("adding bool field to codec: %w", err)
}
case "int":
if err = codec.AddIntField(field.name, lookup); err != nil {
if err = codec.AddIntField(field.name, keys); err != nil {
return fmt.Errorf("adding int field to codec: %w", err)
}
case "decimal":
@ -1894,17 +1932,53 @@ func (api *API) IngestOperations(ctx context.Context, qcx *Qcx, indexName string
if err != nil {
return errors.Wrap(err, "parsing input data")
}
sharded, err := req.ByShard()
sharded, err := codec.RequestByShard(req)
if err != nil {
return errors.Wrap(err, "sharding input data")
}
eg, ctx := errgroup.WithContext(ctx)
// now that we have this, let's assign the shards to nodes
snap := topology.NewClusterSnapshot(api.cluster.noder, api.cluster.Hasher, api.cluster.ReplicaN)
// oh hey an easy case: we're presumably the only node
if len(snap.Nodes) == 1 {
return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded)
}
// Created new ShardedRequest objects for every node, giving each of them
// all the shards that apply to them.
byNode := make(map[string]*ingest.ShardedRequest)
for shard, ops := range sharded.Ops {
// loop variable shadow capture is the go equivalent of man door hook hand
shard, ops := shard, ops
eg.Go(func() error {
return api.applyOperations(ctx, qcx, index, shard, knownFields, ops)
})
nodes := snap.ShardNodes(indexName, shard)
for _, node := range nodes {
forThisShard := byNode[node.ID]
if forThisShard == nil {
// Create new ShardedRequest for the target node, with its op map
// mapping this shard to the ops for this shard.
byNode[node.ID] = &ingest.ShardedRequest{Ops: map[uint64][]*ingest.Operation{shard: ops}}
continue
}
// Add this shard to the existing ShardedRequest's Ops map. Note that
// we don't have to worry about overwrites; we can't have seen this
// shard before, because we're in a range loop on a map where the shard
// is the key.
forThisShard.Ops[shard] = ops
}
}
eg, ctx := errgroup.WithContext(ctx)
for _, node := range snap.Nodes {
node := node
sharded := byNode[node.ID]
// Sometimes, there's nothing for a specific node.
if sharded == nil {
continue
}
if node.ID == api.NodeID() {
eg.Go(func() error {
return api.ingestNodeOperationsForFields(ctx, qcx, index, knownFields, sharded)
})
} else {
eg.Go(func() error {
return api.server.defaultClient.IngestNodeOperations(ctx, &node.URI, indexName, sharded)
})
}
}
return eg.Wait()
}
@ -2692,9 +2766,9 @@ func mergeIDLists(dst []uint64, src []uint64) []uint64 {
return dst[i] < dst[j]
})
// dedup.
n := 0
n := 1
prev := dst[0]
for i := 0; i < len(dst); i++ {
for i := 1; i < len(dst); i++ {
if dst[i] != prev {
dst[n] = dst[i]
n++
@ -2712,9 +2786,9 @@ func mergeKeyLists(dst []string, src []string) []string {
return dst[i] < dst[j]
})
// dedup.
n := 0
n := 1
prev := dst[0]
for i := 0; i < len(dst); i++ {
for i := 1; i < len(dst); i++ {
if dst[i] != prev {
dst[n] = dst[i]
n++
@ -3024,6 +3098,9 @@ processing:
}
return result, ctx.Err()
}
func (api *API) Plan(ctx context.Context, q string) (*Stmt, error) {
return api.server.PlanSQL(ctx, q)
}
type serverInfo struct {
ShardWidth uint64 `json:"shardWidth"`
@ -3085,6 +3162,7 @@ const (
apiIDReset
apiPartitionNodes
apiIngestOperations
apiIngestNodeOperations
apiMutexCheck
)
@ -3154,5 +3232,6 @@ var methodsNormal = map[apiMethod]struct{}{
apiIDReset: {},
apiPartitionNodes: {},
apiIngestOperations: {},
apiIngestNodeOperations: {},
apiMutexCheck: {},
}

8
api_test.go
View file

@ -860,7 +860,13 @@ type mutexCheckField struct {
}
func TestAPI_MutexCheck(t *testing.T) {
c := test.MustRunCluster(t, 3)
c := test.MustNewCluster(t, 3)
for _, c := range c.Nodes {
c.Config.Cluster.ReplicaN = 2
}
if err := c.Start(); err != nil {
t.Fatalf("starting cluster: %v", err)
}
defer c.Close()
m0 := c.GetNode(0)

10
bolt.go
View file

@ -135,8 +135,12 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag
if !DirExists(path) {
PanicOn(os.MkdirAll(dir, 0755))
}
fsyncEnabled := true
if cfg != nil {
fsyncEnabled = cfg.FsyncEnabled
}
db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize})
db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !fsyncEnabled})
if err != nil {
return nil, errors.Wrapf(err, fmt.Sprintf("open bolt path '%v'", path))
}
@ -187,6 +191,7 @@ func (r *boltRegistrar) OpenDBWrapper(path string, doAllocZero bool, cfg *storag
openTx: make(map[*BoltTx]bool),
DeleteEmptyContainer: true,
fsyncEnabled: cfg.FsyncEnabled,
}
r.unprotectedRegister(w)
@ -226,7 +231,7 @@ func (w *BoltWrapper) CloseDB() error {
func (w *BoltWrapper) OpenDB() error {
w.muDb.Lock()
defer w.muDb.Unlock()
db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize})
db, err := bolt.Open(w.path, 0666, &bolt.Options{Timeout: 5 * time.Second, InitialMmapSize: TxInitialMmapSize, NoSync: !w.fsyncEnabled})
if err != nil {
return err
}
@ -315,6 +320,7 @@ type BoltWrapper struct {
doAllocZero bool
DeleteEmptyContainer bool
fsyncEnabled bool // for tracking whether our initial config wanted fsync on
openTx map[*BoltTx]bool
}

3
bolt_test.go
View file

@ -21,6 +21,7 @@ import (
"testing"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/storage"
. "github.com/molecula/featurebase/v2/vprint" // nolint:staticcheck
)
@ -88,7 +89,7 @@ func mustOpenEmptyBoltWrapper(path string) (w *BoltWrapper, cleaner func()) {
var err error
fn := path
PanicOn(os.RemoveAll(fn))
ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, nil)
ww, err := globalBoltReg.OpenDBWrapper(fn, DetectMemAccessPastTx, &storage.Config{FsyncEnabled: false})
PanicOn(err)
w = ww.(*BoltWrapper)

26
boltdb/translate.go
View file

@ -55,8 +55,8 @@ const (
)
// OpenTranslateStore opens and initializes a boltdb translation store.
func OpenTranslateStore(path, index, field string, partitionID, partitionN int) (pilosa.TranslateStore, error) {
s := NewTranslateStore(index, field, partitionID, partitionN)
func OpenTranslateStore(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (pilosa.TranslateStore, error) {
s := NewTranslateStore(index, field, partitionID, partitionN, fsyncEnabled)
s.Path = path
if err := s.Open(); err != nil {
return nil, err
@ -88,22 +88,24 @@ type TranslateStore struct {
once sync.Once
closing chan struct{}
readOnly bool
writeNotify chan struct{}
readOnly bool
fsyncEnabled bool
writeNotify chan struct{}
// File path to database file.
Path string
}
// NewTranslateStore returns a new instance of TranslateStore.
func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore {
func NewTranslateStore(index, field string, partitionID, partitionN int, fsyncEnabled bool) *TranslateStore {
return &TranslateStore{
index: index,
field: field,
partitionID: partitionID,
partitionN: partitionN,
closing: make(chan struct{}),
writeNotify: make(chan struct{}),
index: index,
field: field,
partitionID: partitionID,
partitionN: partitionN,
closing: make(chan struct{}),
writeNotify: make(chan struct{}),
fsyncEnabled: fsyncEnabled,
}
}
@ -120,7 +122,7 @@ func (s *TranslateStore) Open() (err error) {
if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil {
return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path))
} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil {
} else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !s.fsyncEnabled}); err != nil {
return errors.Wrapf(err, "open file: %s", err)
}

2
boltdb/translate_test.go
View file

@ -393,7 +393,7 @@ func MustNewTranslateStore(tb testing.TB) *boltdb.TranslateStore {
panic(err)
}
s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN)
s := boltdb.NewTranslateStore("I", "F", 0, topology.DefaultPartitionN, false)
s.Path = f.Name()
return s
}

6
client.go
View file

@ -19,6 +19,7 @@ import (
"io"
"time"
"github.com/molecula/featurebase/v2/ingest"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/topology"
)
@ -81,6 +82,7 @@ type InternalClient interface {
ImportRoaring(ctx context.Context, uri *pnet.URI, index, field string, shard uint64, remote bool, req *ImportRoaringRequest) error
ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error)
MutexCheck(ctx context.Context, uri *pnet.URI, index string, field string, details bool, limit int) (map[uint64]map[uint64][]uint64, error)
IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error
IDAllocDataReader(ctx context.Context) (io.ReadCloser, error)
IndexTranslateDataReader(ctx context.Context, index string, partitionID int) (io.ReadCloser, error)
@ -218,6 +220,10 @@ func (n nopInternalClient) MutexCheck(ctx context.Context, uri *pnet.URI, index,
return nil, nil
}
func (n nopInternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error {
return nil
}
func (n nopInternalClient) ShardReader(ctx context.Context, index string, shard uint64) (io.ReadCloser, error) {
return nil, nil
}

38
cluster.go
View file

@ -24,6 +24,7 @@ import (
"time"
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/roaring"
"github.com/molecula/featurebase/v2/topology"
@ -1165,6 +1166,9 @@ func (c *cluster) followResizeInstruction(ctx context.Context, instr *ResizeInst
}
}
// fire off translation sync
_ = c.translationSyncer.Reset()
return nil
}
@ -1179,6 +1183,8 @@ func (c *cluster) resizeAbort() error {
if c.resizeCancel != nil {
c.resizeCancel()
}
// fire off translation sync
_ = c.translationSyncer.Reset()
return nil
}
@ -1561,6 +1567,38 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys
return ids, nil
}
// This implements ingest's key translator interface on a cluster/index pair.
type clusterKeyTranslator struct {
ctx context.Context // we're created within a request context and need to pass that to cluster ops
c *cluster
indexName string
}
var _ ingest.KeyTranslator = &clusterKeyTranslator{}
func (i clusterKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
return i.c.createIndexKeys(i.ctx, i.indexName, keys...)
}
func (i clusterKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
keys, err := i.c.translateIndexIDs(i.ctx, i.indexName, ids)
if err != nil {
return nil, err
}
if len(keys) != len(ids) {
return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys))
}
out := make(map[uint64]string, len(keys))
for i, id := range ids {
out[id] = keys[i]
}
return out, nil
}
func newIngestKeyTranslatorFromCluster(ctx context.Context, c *cluster, indexName string) *clusterKeyTranslator {
return &clusterKeyTranslator{ctx: ctx, c: c, indexName: indexName}
}
// TODO: remove this when it is no longer used
func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) {
keys := make([]string, 0, len(keySet))

5
cluster_internal_test.go
View file

@ -155,7 +155,10 @@ func newIndexWithTempPath(tb testing.TB, name string) *Index {
if err != nil {
panic(err)
}
h := NewHolder(path, nil)
cfg := DefaultHolderConfig()
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
h := NewHolder(path, cfg)
PanicOn(h.Open())
index, err := h.CreateIndex(name, IndexOptions{})
testhook.Cleanup(tb, func() {

1
cmd/backup.go
View file

@ -40,6 +40,7 @@ Backs up a FeatureBase server to a local, tar-formatted snapshot file.
flags.BoolVar(&cmd.NoSync, "no-sync", false, "disable file sync")
flags.IntVar(&cmd.Concurrency, "concurrency", cmd.Concurrency, "number of concurrent backup goroutines")
flags.StringVar(&cmd.Host, "host", "localhost:10101", "host:port of FeatureBase.")
flags.StringVar(&cmd.Index, "index", "", "index to backup, default backs up all indexes. ")
ctl.SetTLSConfig(flags, "", &cmd.TLS.CertificatePath, &cmd.TLS.CertificateKeyPath, &cmd.TLS.CACertPath, &cmd.TLS.SkipVerify, &cmd.TLS.EnableClientVerification)
return ccmd
}

27
cmd/roaring-migrate/ctim_darwin.go Normal file
View file

@ -0,0 +1,27 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build darwin
package main
import (
"syscall"
)
func CTimeNano(stat *syscall.Stat_t) int64 {
NANOS := int64(1e9) // number of nanosecs in 1 sec
ts := stat.Ctimespec
return ts.Sec*NANOS + ts.Nsec
}

14
ingest/translate.go → cmd/roaring-migrate/ctim_linux.go
View file

@ -1,4 +1,4 @@
// Copyright 2021 Molecula Corp.
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -12,4 +12,14 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package ingest
// +build linux
package main
import (
"syscall"
)
func CTimeNano(stat *syscall.Stat_t) int64 {
return stat.Ctim.Nano()
}

6
cmd/roaring-migrate/main.go
View file

@ -24,7 +24,7 @@ import (
"strings"
"syscall"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/rbf"
"github.com/molecula/featurebase/v2/rbf/cfg"
"github.com/molecula/featurebase/v2/roaring"
@ -148,7 +148,7 @@ func BuildSchema(dataDir string) ([]byte, error) {
l = &local{
Name: index,
Fields: make([]*pilosa.FieldInfo, 0),
CreatedAt: uint64(stat.Ctim.Nano()),
CreatedAt: uint64(CTimeNano(stat)),
Options: *io,
}
//index options
@ -158,7 +158,7 @@ func BuildSchema(dataDir string) ([]byte, error) {
field := t[2]
if field != "_exists" {
fi, err := pilosa.UnmarshalFieldOptions(field, stat.Ctim.Nano(), content)
fi, err := pilosa.UnmarshalFieldOptions(field, CTimeNano(stat), content)
if err != nil {
return err
}

18
ctl/backup.go
View file

@ -24,7 +24,7 @@ import (
"os"
"path/filepath"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/topology"
@ -38,6 +38,9 @@ type BackupCommand struct { // nolint: maligned
// Destination host and port.
Host string `json:"host"`
// Optional Index filter
Index string `json:"index"`
// Path to write the backup to.
OutputDir string
@ -91,6 +94,19 @@ func (cmd *BackupCommand) Run(ctx context.Context) (err error) {
if err != nil {
return fmt.Errorf("getting schema: %w", err)
}
if cmd.Index != "" {
for _, idx := range indexes {
if idx.Name == cmd.Index {
indexes = make([]*pilosa.IndexInfo, 0)
indexes = append(indexes, idx)
break
}
}
if len(indexes) <= 0 {
return fmt.Errorf("Index not found to back up")
}
}
schema := &pilosa.Schema{Indexes: indexes}
// Ensure output directory doesn't exist; then create output directory.

50
ctl/restore.go
View file

@ -17,6 +17,7 @@ package ctl
import (
"context"
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
@ -26,7 +27,7 @@ import (
"strconv"
"strings"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/topology"
"golang.org/x/sync/errgroup"
@ -102,7 +103,6 @@ func (cmd *RestoreCommand) Run(ctx context.Context) (err error) {
} else if err := cmd.restoreIDAlloc(ctx, primary); err != nil {
return fmt.Errorf("cannot restore idalloc: %w", err)
}
if err := cmd.restoreShards(ctx); err != nil {
return fmt.Errorf("cannot restore shards: %w", err)
} else if err := cmd.restoreIndexTranslation(ctx); err != nil {
@ -128,11 +128,47 @@ func (cmd *RestoreCommand) restoreSchema(ctx context.Context, primary *topology.
}
defer f.Close()
cmd.Logger().Printf("Load Schema")
url := primary.URI.Path("/schema")
var client http.Client
_, err = client.Post(url, "application/json", f)
existingSchema, err := cmd.client.Schema(ctx)
if len(existingSchema) == 0 {
cmd.Logger().Printf("Load Schema")
url := primary.URI.Path("/schema")
var client http.Client
_, err = client.Post(url, "application/json", f)
} else {
schema := &pilosa.Schema{}
if err := json.NewDecoder(f).Decode(schema); err != nil {
if err != nil {
return err
}
}
exists := func(indexName string) bool {
for _, i := range existingSchema {
if i.Name == indexName {
return true
}
}
return false
}
logger := cmd.Logger()
//NOTE SHOULD ONLY BE ONE
for _, index := range schema.Indexes {
if exists(index.Name) {
return fmt.Errorf("Index Exists %v", index.Name)
}
logger.Printf("Create INDEX %v", index.Name)
err = cmd.client.CreateIndex(ctx, index.Name, index.Options)
if err != nil {
return err
}
for _, field := range index.Fields {
logger.Printf("Create Field %v", field.Name)
err = cmd.client.CreateFieldWithOptions(ctx, index.Name, field.Name, field.Options)
if err != nil {
return err
}
}
}
}
return err
}

1
ctl/server.go
View file

@ -111,6 +111,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) {
flags.DurationVar((*time.Duration)(&srv.Config.Postgres.WriteTimeout), "postgres.write-timeout", time.Duration(srv.Config.Postgres.WriteTimeout), "Timeout for writes on a postgres connection. (set 0 to disable)")
flags.Uint32Var(&srv.Config.Postgres.MaxStartupSize, "postgres.max-startup-size", srv.Config.Postgres.MaxStartupSize, "Maximum acceptable size of a postgres startup packet, in bytes. (set 0 to disable)")
flags.Uint16Var(&srv.Config.Postgres.ConnectionLimit, "postgres.connection-limit", srv.Config.Postgres.ConnectionLimit, "Maximum number of simultaneous postgres connections to allow. (set 0 to disable)")
flags.Uint16Var(&srv.Config.Postgres.SqlVersion, "postgres.sql-version", srv.Config.Postgres.SqlVersion, "Molecula Sql Handling Version (default 1)")
// Disk and Memory usage cache for ui/usage endpoint
flags.Float64Var(&srv.Config.UsageDutyCycle, "usage-duty-cycle", srv.Config.UsageDutyCycle, "Sets the percentage of time that is spent recalculating the disk and memory usage cache. 100.0 for always-running, 0 disables the cache and the /ui/usage endpoint.")

1
dbshard_internal_test.go
View file

@ -330,6 +330,7 @@ func Test_DBPerShard_GetFieldView2Shards_map_from_RBF(t *testing.T) {
cfg := mustHolderConfig()
cfg.StorageConfig.Backend = "rbf"
cfg.StorageConfig.FsyncEnabled = false
holder := NewHolder(tmpdir, cfg)
defer holder.Close()

112
encoding/proto/proto.go
View file

@ -21,6 +21,7 @@ import (
"github.com/gogo/protobuf/proto"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/ingest"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/pb"
"github.com/molecula/featurebase/v2/pql"
@ -324,7 +325,18 @@ func (s Serializer) Unmarshal(buf []byte, m pilosa.Message) error {
}
decodeResizeAbortMessage(msg, mt)
return nil
case *ingest.ShardedRequest:
msg := &pb.ShardedIngestRequest{}
err := proto.Unmarshal(buf, msg)
if err != nil {
return errors.Wrap(err, "unmarshalling ShardedRequest")
}
req, err := s.decodeShardedIngestRequest(msg)
if err != nil {
return err
}
*mt = *req
return nil
default:
panic(fmt.Sprintf("unhandled pilosa.Message of type %T: %#v", mt, m))
}
@ -398,6 +410,8 @@ func (s Serializer) encodeToProto(m pilosa.Message) proto.Message {
return s.encodeResizeNodeMessage(mt)
case *pilosa.ResizeAbortMessage:
return s.encodeResizeAbortMessage(mt)
case *ingest.ShardedRequest:
return s.encodeShardedIngestRequest(mt)
}
return nil
}
@ -931,6 +945,48 @@ func (s Serializer) encodeTransactionStats(stats pilosa.TransactionStats) *pb.Tr
return &pb.TransactionStats{}
}
func (s Serializer) encodeShardedIngestRequest(req *ingest.ShardedRequest) *pb.ShardedIngestRequest {
if req == nil || len(req.Ops) == 0 {
return &pb.ShardedIngestRequest{}
}
out := &pb.ShardedIngestRequest{Ops: make(map[uint64]*pb.ShardIngestOperations, len(req.Ops))}
for shard, ops := range req.Ops {
out.Ops[shard] = s.encodeShardIngestOperations(ops)
}
return out
}
func (s Serializer) encodeShardIngestOperations(ops []*ingest.Operation) *pb.ShardIngestOperations {
out := &pb.ShardIngestOperations{}
for _, op := range ops {
if op == nil {
continue
}
out.Ops = append(out.Ops, s.encodeShardIngestOperation(op))
}
return out
}
func (s Serializer) encodeShardIngestOperation(op *ingest.Operation) *pb.ShardIngestOperation {
out := &pb.ShardIngestOperation{
OpType: op.OpType.String(),
ClearRecordIDs: op.ClearRecordIDs,
ClearFields: op.ClearFields,
FieldOps: make(map[string]*pb.FieldOperation, len(op.FieldOps)),
}
for k, v := range op.FieldOps {
if v == nil {
continue
}
out.FieldOps[k] = &pb.FieldOperation{
RecordIDs: v.RecordIDs,
Values: v.Values,
Signed: v.Signed,
}
}
return out
}
func (s Serializer) decodeResizeInstruction(ri *pb.ResizeInstruction, m *pilosa.ResizeInstruction) {
m.JobID = ri.JobID
m.Node = &topology.Node{}
@ -1829,3 +1885,57 @@ func decodeResizeNodeMessage(pb *pb.ResizeNodeMessage, m *pilosa.ResizeNodeMessa
func decodeResizeAbortMessage(pb *pb.ResizeAbortMessage, m *pilosa.ResizeAbortMessage) {
}
func (s Serializer) decodeShardedIngestRequest(req *pb.ShardedIngestRequest) (*ingest.ShardedRequest, error) {
if req == nil || len(req.Ops) == 0 {
return &ingest.ShardedRequest{}, nil
}
out := &ingest.ShardedRequest{Ops: make(map[uint64][]*ingest.Operation, len(req.Ops))}
for shard, ops := range req.Ops {
var err error
out.Ops[shard], err = s.decodeShardIngestOperations(ops)
if err != nil {
return nil, err
}
}
return out, nil
}
func (s Serializer) decodeShardIngestOperations(ops *pb.ShardIngestOperations) ([]*ingest.Operation, error) {
out := []*ingest.Operation{}
if len(ops.Ops) == 0 {
return out, nil
}
for _, op := range ops.Ops {
if op == nil {
continue
}
decoded, err := s.decodeShardIngestOperation(op)
if err != nil {
return nil, err
}
out = append(out, decoded)
}
return out, nil
}
func (s Serializer) decodeShardIngestOperation(op *pb.ShardIngestOperation) (*ingest.Operation, error) {
opType, err := ingest.ParseOpType(op.OpType)
if err != nil {
return nil, err
}
out := &ingest.Operation{
OpType: opType,
ClearRecordIDs: op.ClearRecordIDs,
ClearFields: op.ClearFields,
FieldOps: make(map[string]*ingest.FieldOperation, len(op.FieldOps)),
}
for k, v := range op.FieldOps {
out.FieldOps[k] = &ingest.FieldOperation{
RecordIDs: v.RecordIDs,
Values: v.Values,
Signed: v.Signed,
}
}
return out, nil
}

149
encoding/proto/proto_test.go Normal file
View file

@ -0,0 +1,149 @@
// Copyright 2021 Molecula Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package proto
import (
"errors"
"reflect"
"testing"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/ingest"
)
func testOneRoundTrip(t *testing.T, s pilosa.Serializer, obj pilosa.Message, expectedMarshalErr error, expectedUnmarshalErr error, expectedMismatchErr error) {
repr, err := s.Marshal(obj)
if err != nil {
if expectedMarshalErr == nil {
t.Fatalf("unexpected marshalling error %q", err.Error())
}
if err.Error() != expectedMarshalErr.Error() {
t.Fatalf("expecting marshalling error %q, got %q", expectedMarshalErr.Error(), err.Error())
}
} else {
if expectedMarshalErr != nil {
t.Fatalf("expected marshalling error %q, got no error", expectedMarshalErr.Error())
}
}
obj2 := reflect.New(reflect.TypeOf(obj).Elem()).Interface()
err = s.Unmarshal(repr, obj2)
if err != nil {
if expectedUnmarshalErr == nil {
t.Fatalf("unexpected unmarshalling error %q", err.Error())
}
if err.Error() != expectedUnmarshalErr.Error() {
t.Fatalf("expecting unmarshalling error %q, got %q", expectedUnmarshalErr.Error(), err.Error())
}
} else {
if expectedUnmarshalErr != nil {
t.Fatalf("expected unmarshalling error %q, got no error", expectedUnmarshalErr.Error())
}
}
switch real := obj.(type) {
case *ingest.ShardedRequest:
real2 := obj2.(*ingest.ShardedRequest)
err := real.Compare(real2)
if err != nil {
if expectedMismatchErr == nil {
t.Fatalf("unexpected compare error %q", err.Error())
}
if err.Error() != expectedMismatchErr.Error() {
t.Fatalf("expecting compare error %q, got %q", expectedMismatchErr.Error(), err.Error())
}
} else {
if expectedMismatchErr != nil {
t.Fatalf("expected compare error %q, got no error", expectedMismatchErr.Error())
}
}
default:
if !reflect.DeepEqual(obj, obj2) {
t.Fatalf("serialization round trip failed for %T:\nexpected %#v\ngot %#v", obj, obj, obj2)
}
}
}
type shardedIngestRequestTest struct {
req *ingest.ShardedRequest
err error
}
var shardedIngestRequestTestcases = []shardedIngestRequestTest{
{
req: &ingest.ShardedRequest{
Ops: map[uint64][]*ingest.Operation{
1: {
{
OpType: ingest.OpWrite,
ClearFields: []string{"clearField", "clearField2"},
ClearRecordIDs: []uint64{1, 7, 9},
FieldOps: map[string]*ingest.FieldOperation{
"writeAll": {
RecordIDs: []uint64{3, 6, 8},
Values: []uint64{0, 17, 34},
Signed: []int64{-9, 23, 17},
},
"writeValues": {
RecordIDs: []uint64{3, 6, 8},
Values: []uint64{0, 17, 34},
},
"writeSigned": {
RecordIDs: []uint64{3, 6, 8},
Signed: []int64{-9, 23, 17},
},
},
},
{
OpType: ingest.OpSet,
FieldOps: map[string]*ingest.FieldOperation{
"foo": nil,
},
},
},
2: {},
3: nil,
},
},
},
{
req: &ingest.ShardedRequest{
Ops: map[uint64][]*ingest.Operation{
1: {
{
OpType: ingest.OpWrite,
FieldOps: map[string]*ingest.FieldOperation{
"writeAll": {},
},
},
{
OpType: ingest.OpSet,
FieldOps: map[string]*ingest.FieldOperation{
"foo": nil,
},
},
nil,
},
},
},
err: errors.New("shard 1: expected 3 ops, got 2"),
},
}
func TestIngestRoundTrip(t *testing.T) {
for _, tc := range shardedIngestRequestTestcases {
t.Logf("next case")
testOneRoundTrip(t, DefaultSerializer, tc.req, nil, nil, tc.err)
}
}

2
etcd/embed.go
View file

@ -63,6 +63,7 @@ type Options struct {
LClientSocket []*net.TCPListener
BootstrapTimeout time.Duration
UnsafeNoFsync bool `toml:"no-fsync"`
}
var (
@ -228,6 +229,7 @@ func parseOptions(opt Options) *embed.Config {
cfg.InitialClusterToken = opt.ClusterName
cfg.BootstrapTimeout = opt.BootstrapTimeout
cfg.LCUrls = types.MustNewURLs([]string{opt.LClientURL})
cfg.UnsafeNoFsync = opt.UnsafeNoFsync
if opt.AClientURL != "" {
cfg.ACUrls = types.MustNewURLs([]string{opt.AClientURL})
} else {

89
executor.go
View file

@ -1825,10 +1825,14 @@ func (e *executor) executeSumCountShard(ctx context.Context, qcx *Qcx, index str
if err != nil {
return ValCount{}, errors.Wrap(err, "computing sum")
}
return ValCount{
out := ValCount{
Val: int64(vsum) + (int64(vcount) * bsig.Base),
Count: int64(vcount),
}, nil
}
if field.Type() == FieldTypeDecimal {
out.FloatVal = float64(int64(vsum)+(int64(vcount)*bsig.Base)) / math.Pow(10, float64(bsig.Scale))
}
return out, nil
}
// executeMinShard calculates the min for bsiGroups on a shard.
@ -3044,6 +3048,13 @@ func (e *executor) executeGroupBy(ctx context.Context, qcx *Qcx, index string, c
aggType = "aggregate"
}
}
for _, res := range results {
if res.DecimalAgg != 0 && aggType == "sum" {
aggType = "decimalSum"
break
}
}
return NewGroupCounts(aggType, results...), nil
}
@ -3132,9 +3143,10 @@ func (fr FieldRow) String() string {
type aggregateType int
const (
nilAggregate aggregateType = 0
sumAggregate aggregateType = 1
distinctAggregate aggregateType = 2
nilAggregate aggregateType = 0
sumAggregate aggregateType = 1
distinctAggregate aggregateType = 2
decimalSumAggregate aggregateType = 3
)
// GroupCounts is a list of GroupCount.
@ -3152,6 +3164,8 @@ func (g *GroupCounts) AggregateColumn() string {
return "sum"
case distinctAggregate:
return "aggregate"
case decimalSumAggregate:
return "decimalSum"
default:
return ""
}
@ -3176,6 +3190,8 @@ func NewGroupCounts(agg string, groups ...GroupCount) *GroupCounts {
aggType = sumAggregate
case "aggregate":
aggType = distinctAggregate
case "decimalSum":
aggType = decimalSumAggregate
case "":
aggType = nilAggregate
default:
@ -3246,42 +3262,57 @@ func (g *GroupCounts) MarshalJSON() ([]byte, error) {
if len(groups) == 0 {
return []byte("[]"), nil
}
switch g.aggregateType {
case sumAggregate:
counts = *(*[]groupCountSum)(unsafe.Pointer(&groups))
case distinctAggregate:
counts = *(*[]groupCountAggregate)(unsafe.Pointer(&groups))
case decimalSumAggregate:
counts = *(*[]groupCountDecimalSum)(unsafe.Pointer(&groups))
}
return json.Marshal(counts)
}
// GroupCount represents a result item for a group by query.
type GroupCount struct {
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Agg int64 `json:"-"`
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Agg int64 `json:"-"`
DecimalAgg float64 `json:"-"`
}
type groupCountSum struct {
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Agg int64 `json:"sum"`
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Agg int64 `json:"sum"`
DecimalAgg float64 `json:"-"`
}
type groupCountAggregate struct {
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Agg int64 `json:"aggregate"`
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Agg int64 `json:"aggregate"`
DecimalAgg float64 `json:"-"`
}
type groupCountDecimalSum struct {
Group []FieldRow `json:"group"`
Count uint64 `json:"count"`
Agg int64 `json:"-"`
DecimalAgg float64 `json:"sum"`
}
var _ GroupCount = GroupCount(groupCountSum{})
var _ GroupCount = GroupCount(groupCountAggregate{})
var _ GroupCount = GroupCount(groupCountDecimalSum{})
func (g *GroupCount) Clone() (r *GroupCount) {
r = &GroupCount{
Group: make([]FieldRow, len(g.Group)),
Count: g.Count,
Agg: g.Agg,
Group: make([]FieldRow, len(g.Group)),
Count: g.Count,
Agg: g.Agg,
DecimalAgg: g.DecimalAgg,
}
for i := range g.Group {
r.Group[i] = *(g.Group[i].Clone())
@ -3306,6 +3337,7 @@ func mergeGroupCounts(a, b []GroupCount, limit int) []GroupCount {
case 0:
a[i].Count += b[j].Count
a[i].Agg += b[j].Agg
a[i].DecimalAgg += b[j].DecimalAgg
ret = append(ret, a[i])
i++
j++
@ -4718,25 +4750,25 @@ func (e *executor) executeIntersectShard(ctx context.Context, qcx *Qcx, index st
}
// executeUnionShard executes a union() call for a local shard.
func (e *executor) executeUnionShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (_ *Row, err error) {
func (e *executor) executeUnionShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (out *Row, err error) {
span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeUnionShard")
defer span.Finish()
other := NewRow()
if len(c.Children) == 0 {
return NewRow(), nil
}
if len(c.Children) == 1 {
return e.executeBitmapCallShard(ctx, qcx, index, c.Children[0], shard)
}
// we have at least two, so...
rows := make([]*Row, len(c.Children))
for i, input := range c.Children {
row, err := e.executeBitmapCallShard(ctx, qcx, index, input, shard)
rows[i], err = e.executeBitmapCallShard(ctx, qcx, index, input, shard)
if err != nil {
return nil, err
}
if i == 0 {
other = row
} else {
other = other.Union(row)
}
}
other.invalidateCount()
return other, nil
return rows[0].Union(rows[1:]...), nil
}
// executeXorShard executes a xor() call for a local shard.
@ -7939,6 +7971,7 @@ func (gbi *groupByIterator) Next(ctx context.Context) (ret GroupCount, done bool
}
ret.Count = uint64(result.Count)
ret.Agg = result.Val
ret.DecimalAgg = result.FloatVal
}
}
if ret.Count == 0 {

2
executor_internal_test.go
View file

@ -28,7 +28,7 @@ import (
func TestExecutor_TranslateRowsOnBool(t *testing.T) {
path, _ := testhook.TempDirInDir(t, *TempDir, "pilosa-executor-")
holder := NewHolder(path, nil)
holder := NewHolder(path, mustHolderConfig())
defer holder.Close()
e := &executor{

42
executor_test.go
View file

@ -38,7 +38,7 @@ import (
"github.com/davecgh/go-spew/spew"
"github.com/google/go-cmp/cmp"
"github.com/google/go-cmp/cmp/cmpopts"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/boltdb"
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/http"
@ -5151,6 +5151,8 @@ func TestExecutor_GroupByStrings(t *testing.T) {
c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "v", pilosa.OptFieldTypeInt(0, 1000))
c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "vv", pilosa.OptFieldTypeInt(0, 1000))
c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "nv", pilosa.OptFieldTypeInt(-1000, 1000))
c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "dv", pilosa.OptFieldTypeDecimal(2))
c.CreateField(t, "istring", pilosa.IndexOptions{Keys: true}, "ndv", pilosa.OptFieldTypeDecimal(1))
if err := c.GetNode(0).API.Import(context.Background(), nil, &pilosa.ImportRequest{
Index: "istring",
@ -5167,6 +5169,8 @@ func TestExecutor_GroupByStrings(t *testing.T) {
var v1, v2, v3, v4, v5, v6, v7, v8, v9, v10 int64 = 1, 2, 3, 4, 5, 6, 7, 8, 9, 10
var nv1, nv2, nv3, nv4 int64 = -1, -2, -3, -4
var dv1, dv2, dv3, dv4, dv5, dv6, dv7, dv8, dv9, dv10 int64 = 111, 222, 333, 444, 555, 666, 777, 888, 999, 1000
var ndv1, ndv2, ndv3, ndv4, ndv5, ndv6, ndv7, ndv8, ndv9, ndv10 int64 = -111, -222, -333, -444, -555, -666, -777, -888, -999, -1000
if err := m0.API.ImportValue(context.Background(), qcx, &pilosa.ImportValueRequest{
Index: "istring",
Field: "v",
@ -5197,6 +5201,26 @@ func TestExecutor_GroupByStrings(t *testing.T) {
t.Fatalf("importing: %v", err)
}
if err := m0.API.ImportValue(context.Background(), qcx, &pilosa.ImportValueRequest{
Index: "istring",
Field: "dv",
Shard: 0,
ColumnKeys: []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10"},
Values: []int64{dv1, dv2, dv3, dv4, dv5, dv6, dv7, dv8, dv9, dv10},
}); err != nil {
t.Fatalf("importing: %v", err)
}
if err := m0.API.ImportValue(context.Background(), qcx, &pilosa.ImportValueRequest{
Index: "istring",
Field: "ndv",
Shard: 0,
ColumnKeys: []string{"c1", "c2", "c3", "c4", "c5", "c6", "c7", "c8", "c9", "c10"},
Values: []int64{ndv1, ndv2, ndv3, ndv4, ndv5, ndv6, ndv7, ndv8, ndv9, ndv10},
}); err != nil {
t.Fatalf("importing: %v", err)
}
tests := []struct {
query string
expected []pilosa.GroupCount
@ -5221,6 +5245,20 @@ func TestExecutor_GroupByStrings(t *testing.T) {
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 30},
},
},
{
query: "GroupBy(Rows(generals), aggregate=Sum(field=dv))",
expected: []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: 2775, DecimalAgg: 27.75},
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: 3220, DecimalAgg: 32.20},
},
},
{
query: "GroupBy(Rows(generals), aggregate=Sum(field=ndv))",
expected: []pilosa.GroupCount{
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 1, RowKey: "r1"}}, Count: 5, Agg: -2775, DecimalAgg: -277.5},
{Group: []pilosa.FieldRow{{Field: "generals", RowID: 2, RowKey: "r2"}}, Count: 5, Agg: -3220, DecimalAgg: -322.0},
},
},
{
query: "GroupBy(Rows(generals), aggregate=Sum(field=v), having=Condition(sum>25))",
expected: []pilosa.GroupCount{
@ -6844,7 +6882,7 @@ func TestMissingKeyRegression(t *testing.T) {
c := test.MustRunCluster(t, 1, []server.CommandOption{server.OptCommandServerOptions(
pilosa.OptServerStorageConfig(&storage.Config{
Backend: "roaring",
FsyncEnabled: true,
FsyncEnabled: false,
}))})
defer c.Close()

10
field.go
View file

@ -648,7 +648,7 @@ func (f *Field) writeAvailableShards() {
func (f *Field) applyTranslateStore() error {
// Instantiate & open translation store.
var err error
f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1)
f.translateStore, err = f.OpenTranslateStore(f.TranslateStorePath(), f.index, f.name, -1, -1, f.holder.cfg.StorageConfig.FsyncEnabled)
if err != nil {
return errors.Wrap(err, "opening field translate store")
}
@ -1098,9 +1098,15 @@ func (f *Field) MutexCheck(ctx context.Context, qcx *Qcx, details bool, limit in
if f.Type() != FieldTypeMutex {
return nil, errors.New("mutex check only valid for mutex fields")
}
// Rather than deferring the unlock, we grab the standard view
// from the field's viewMap and unlock immediately. This avoids
// holding the rlock for a potentially long time which blocks any
// write lock, and pending write locks block other read locks.
f.mu.RLock()
defer f.mu.RUnlock()
standard := f.viewMap[viewStandard]
f.mu.RUnlock()
if standard == nil {
// no standard view present means we've never needed to create it,
// so it has no bits set, so it has no extra bits set.

2
field_internal_test.go
View file

@ -244,6 +244,8 @@ func NewTestField(t testing.TB, opts FieldOption) *TestField {
cfg := DefaultHolderConfig()
cfg.StorageConfig.Backend = CurrentBackendOrDefault()
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
h := NewHolder(path, cfg)
PanicOn(h.Open())

6
field_test.go
View file

@ -159,7 +159,7 @@ func TestField_NameRestriction(t *testing.T) {
if err != nil {
panic(err)
}
field, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", ".meta", pilosa.OptFieldTypeDefault())
field, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", ".meta", pilosa.OptFieldTypeDefault())
if field != nil {
t.Fatalf("unexpected field name %s", err)
}
@ -192,13 +192,13 @@ func TestField_NameValidation(t *testing.T) {
panic(err)
}
for _, name := range validFieldNames {
_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
_, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", name, pilosa.OptFieldTypeDefault())
if err != nil {
t.Fatalf("unexpected field name: %s %s", name, err)
}
}
for _, name := range invalidFieldNames {
_, err := pilosa.NewField(pilosa.NewHolder(path, nil), path, "i", name, pilosa.OptFieldTypeDefault())
_, err := pilosa.NewField(pilosa.NewHolder(path, mustHolderConfig()), path, "i", name, pilosa.OptFieldTypeDefault())
if err == nil {
t.Fatalf("expected error on field name: %s", name)
}

4
fragment.go
View file

@ -286,10 +286,6 @@ func (f *fragment) Open() error {
// Fill cache with rows persisted to disk.
f.holder.Logger.Debugf("open cache for index/field/view/fragment: %s/%s/%s/%d", f.index(), f.field(), f.view(), f.shard)
if err := f.openCache(); err != nil {
e2 := f.closeStorage()
if e2 != nil {
return errors.Wrapf(err, "closing storage: %v, after opening cache", e2)
}
return errors.Wrap(err, "opening cache")
}

4
fragment_internal_test.go
View file

@ -3166,7 +3166,7 @@ func BenchmarkImportIntoLargeFragment(b *testing.B) {
origF.Close()
fi.Close()
h := NewHolder(fi.Name(), nil)
h := NewHolder(fi.Name(), mustHolderConfig())
PanicOn(h.Open())
idx, err := h.CreateIndex("i", IndexOptions{})
PanicOn(err)
@ -5146,7 +5146,7 @@ func TestImportClearRestart(t *testing.T) {
PanicOn(tx2.Commit())
h3 := NewHolder(filepath.Dir(f2.path()), nil)
h3 := NewHolder(filepath.Dir(f2.path()), mustHolderConfig())
testhook.Cleanup(t, func() {
h3.Close()
})

4
go.mod
View file

@ -1,6 +1,8 @@
module github.com/molecula/featurebase/v2
replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7
replace go.etcd.io/etcd => github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c
replace go.etcd.io/bbolt => github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554
require (
github.com/CAFxX/gcnotifier v0.0.0-20190112062741-224a280d589d

11
go.sum
View file

@ -230,8 +230,8 @@ github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y=
github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s=
github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7 h1:hufElvtCighE0G2VFJYDGWCY8JlmCWZ1FmXvlf25yUQ=
github.com/molecula/etcd v0.0.0-20210621160528-2cd93f1df0e7/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI=
github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c h1:YnU+8kIrr/7IDGtIYncawklAs54EWihED4DBIm+kjAA=
github.com/molecula/etcd v0.0.0-20210930172242-ad94b354f72c/go.mod h1:1X1h4BZ44WjM0LJof1gKKLap1OA4RsicGCDRtACTkLI=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
@ -285,6 +285,8 @@ github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554 h1:88K0ffxhVphUHxlqW4ewOaXdnJByH4LcCuvYfv0QI/M=
github.com/seebs/bbolt v0.0.0-20210930181431-2ea708af0554/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
github.com/shirou/gopsutil/v3 v3.20.11 h1:NeVf1K0cgxsWz+N3671ojRptdgzvp7BXL3KV21R0JnA=
github.com/shirou/gopsutil/v3 v3.20.11/go.mod h1:igHnfak0qnw1biGeI2qKQvu0ZkwvEkUcCLlYhZzdr/4=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@ -344,9 +346,6 @@ github.com/zeebo/blake3 v0.1.1 h1:Nbsts7DdKThRHHd+YNlqiGlRqGEF2bE2eXN+xQ1hsEs=
github.com/zeebo/blake3 v0.1.1/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4=
github.com/zeebo/pcg v1.0.0 h1:dt+dx+HvX8g7Un32rY9XWoYnd0NmKmrIzpHF7qiTDj0=
github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
@ -443,7 +442,7 @@ golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

155
holder.go
View file

@ -109,7 +109,7 @@ type Holder struct {
OpenTransactionStore OpenTransactionStoreFunc
// Func to open the ID allocator.
OpenIDAllocator func(string) (*idAllocator, error)
OpenIDAllocator func(string, bool) (*idAllocator, error)
// transactionManager
transactionManager *TransactionManager
@ -241,7 +241,7 @@ func DefaultHolderConfig() *HolderConfig {
OpenTranslateStore: OpenInMemTranslateStore,
OpenTranslateReader: nil,
OpenTransactionStore: OpenInMemTransactionStore,
OpenIDAllocator: func(string) (*idAllocator, error) { return &idAllocator{}, nil },
OpenIDAllocator: func(string, bool) (*idAllocator, error) { return &idAllocator{}, nil },
TranslationSyncer: NopTranslationSyncer,
Serializer: GobSerializer,
Schemator: disco.InMemSchemator,
@ -623,7 +623,7 @@ func (h *Holder) Open() error {
h.transactionManager.Log = h.Logger
// Open ID allocator.
h.ida, err = h.OpenIDAllocator(filepath.Join(h.path, "idalloc.db"))
h.ida, err = h.OpenIDAllocator(filepath.Join(h.path, "idalloc.db"), h.cfg.StorageConfig.FsyncEnabled)
if err != nil {
return errors.Wrap(err, "opening ID allocator")
}
@ -1472,7 +1472,10 @@ type holderSyncer struct {
Cluster *cluster
// Translation sync handling.
readers []TranslateEntryReader
readers []TranslateEntryReader
readersMu sync.Mutex
pendingReaders int
stopInitializeReplicationCh chan struct{}
syncers errgroup.Group
@ -1596,6 +1599,23 @@ func (s *holderSyncer) syncFragment(index, field, view string, shard uint64) err
// resetTranslationSync reinitializes streaming sync of translation data.
func (s *holderSyncer) resetTranslationSync() error {
if s.stopInitializeReplicationCh == nil {
// suppose stopTranslationSync[S] holds the lock s.readersMu and tries
// to send a signal to s.stopInitializeRepliationCh. If
// s.stopInitializeReplicationCh is unbufferd and
// s.initializeReplication[I] is trying to acquire s.readersMu, this
// will result in a deadlock.
// Hence, the channel is buffered to prevent this scenario. Once
// [S] releases the lock, [I] will acquire it, however, the value
// of s.pendingReaders will be -1, at this point [I] should not
// attempt to add any more readers since those readers will be 'stale'
// [I] should also drain the channel to prevent the next invocation
// of [I] receiving a stop signal that was meant for the current one
// This is needlessly complicated and is the result of me running
// into various deadlocks while trying to fix handling of column
// key replication.
s.stopInitializeReplicationCh = make(chan struct{})
}
// Stop existing streams.
if err := s.stopTranslationSync(); err != nil {
return errors.Wrap(err, "stop translation sync")
@ -1655,7 +1675,15 @@ func newActiveTranslationSyncer(ch chan struct{}) *activeTranslationSyncer {
// Reset resets the server's translation syncer.
func (a *activeTranslationSyncer) Reset() error {
a.ch <- struct{}{}
// just in case some other part of the code has fired
// off a translation sync and it hasn't been received yet
// therefore we don't want to block on send since a.ch is
// (for now) unbuffered. One translationSync is as good as
// another
select {
case a.ch <- struct{}{}:
default:
}
return nil
}
@ -1665,6 +1693,17 @@ func (a *activeTranslationSyncer) Reset() error {
// to complete. This should be called before reconnecting to the cluster in case
// of a cluster resize or schema change.
func (s *holderSyncer) stopTranslationSync() error {
s.readersMu.Lock()
defer func() {
s.readers = nil // will be populated by initializeReplication
s.readersMu.Unlock()
}()
// send signal to stop initializing more readers
if s.pendingReaders > 0 {
s.pendingReaders = -1
close(s.stopInitializeReplicationCh)
s.stopInitializeReplicationCh = make(chan struct{})
}
var g errgroup.Group
for i := range s.readers {
rd := s.readers[i]
@ -1724,7 +1763,6 @@ func (s *holderSyncer) setTranslateReadOnlyFlags(snap *topology.ClusterSnapshot)
// replicate these from whichever node is primary for that partition).
func (s *holderSyncer) initializeReplication(snap *topology.ClusterSnapshot) error {
nodeMaps := make(map[string]TranslateOffsetMap)
if snap.ReplicaN > 1 {
if err := s.populateIndexReplication(nodeMaps, snap); err != nil {
return err
@ -1734,26 +1772,99 @@ func (s *holderSyncer) initializeReplication(snap *topology.ClusterSnapshot) err
return err
}
// filter out empty nodes
nodes := make(map[*topology.Node]bool)
for _, node := range snap.Nodes {
m := nodeMaps[node.ID]
if m.Empty() {
continue
if !m.Empty() {
nodes[node] = true
}
}
// connect to remote nodes and set up readers
readersCh := make(chan TranslateEntryReader, len(nodes))
ctx, cancelAddingMoreReaders := context.WithCancel(context.Background())
defer cancelAddingMoreReaders()
s.readersMu.Lock()
s.pendingReaders = len(nodes)
s.readersMu.Unlock()
go func() {
for {
for node := range nodes {
// check if ctx cancelled
// this means there was a signal sent to stop further init
// of readers
select {
case <-s.Closing:
return
case <-ctx.Done():
close(readersCh)
return
default:
}
// connect to remote node
m := nodeMaps[node.ID]
rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m)
if err != nil {
continue
}
readersCh <- rd
delete(nodes, node)
}
if len(nodes) == 0 {
close(readersCh)
return
}
time.Sleep(10 * time.Second)
}
}()
for {
select {
case <-s.Closing:
cancelAddingMoreReaders()
return nil
case <-s.stopInitializeReplicationCh:
return nil
case rd, ok := <-readersCh:
// all translate readers have been launched, hence channel is
// closed
if !ok {
return nil
}
s.readersMu.Lock()
// [S] has been initiated and acquired the lock first
// at this point we should close the reader we've recieved rather
// than start replication on it.
// [S] should have already closed all the rest
// of the reads if they were still in action.
// we are also draining the channel since the signal for stopping
// further replication was meant for us
if s.pendingReaders == -1 {
rd.Close()
cancelAddingMoreReaders()
drain:
for {
select {
case <-s.stopInitializeReplicationCh:
default:
break drain
}
}
s.readersMu.Unlock()
return nil
}
s.pendingReaders--
s.readers = append(s.readers, rd)
s.syncers.Go(func() error {
defer rd.Close()
s.readBothTranslateReader(rd, snap)
return nil
})
s.readersMu.Unlock()
}
// Connect to remote node and begin streaming.
rd, err := s.Holder.OpenTranslateReader(context.Background(), node.URI.String(), m)
if err != nil {
return err
}
s.readers = append(s.readers, rd)
s.syncers.Go(func() error {
defer rd.Close()
s.readBothTranslateReader(rd, snap)
return nil
})
}
return nil
}
// populateFieldReplication populates a map from node IDs to TranslateOffsetMaps

3
holder_internal_test.go
View file

@ -86,6 +86,7 @@ func makeHolder(tb testing.TB, backend string) (*Holder, string, error) {
cfg := mustHolderConfig()
if backend != "" {
cfg.StorageConfig.Backend = backend
cfg.StorageConfig.FsyncEnabled = false
}
h := NewHolder(path, cfg)
return h, path, h.Open()
@ -265,6 +266,8 @@ func mustHolderConfig() *HolderConfig {
_ = MustBackendToTxtype(backend)
cfg.StorageConfig.Backend = backend
}
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
cfg.Schemator = disco.InMemSchemator
cfg.Sharder = disco.InMemSharder
return cfg

17
holder_test.go
View file

@ -25,11 +25,26 @@ import (
"time"
"github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/pql"
"github.com/molecula/featurebase/v2/test"
"github.com/pkg/errors"
)
// mustHolderConfig provides a default test-friendly holder config.
func mustHolderConfig() *pilosa.HolderConfig {
cfg := pilosa.DefaultHolderConfig()
if backend := pilosa.CurrentBackend(); backend != "" {
_ = pilosa.MustBackendToTxtype(backend)
cfg.StorageConfig.Backend = backend
}
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
cfg.Schemator = disco.InMemSchemator
cfg.Sharder = disco.InMemSharder
return cfg
}
func TestHolder_Open(t *testing.T) {
t.Run("ErrIndexPermission", func(t *testing.T) {
if os.Geteuid() == 0 {
@ -283,7 +298,7 @@ func TestHolder_HasData(t *testing.T) {
// Note that we are intentionally not using test.NewHolder,
// because we want to create a Holder object with an invalid path,
// rather than creating a valid holder with a temporary path.
h := pilosa.NewHolder("bad-path", nil)
h := pilosa.NewHolder("bad-path", mustHolderConfig())
if ok, err := h.HasData(); ok || err != nil {
t.Fatal("expected HasData to return false, no err, but", ok, err)

33
http/client.go
View file

@ -32,6 +32,7 @@ import (
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/encoding/proto"
"github.com/molecula/featurebase/v2/ingest"
pnet "github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/topology"
"github.com/molecula/featurebase/v2/tracing"
@ -284,6 +285,38 @@ func (c *InternalClient) IngestOperations(ctx context.Context, uri *pnet.URI, in
return nil
}
// IngestNodeOperations uses the internal/protobuf ingest endpoint for ingest data
func (c *InternalClient) IngestNodeOperations(ctx context.Context, uri *pnet.URI, indexName string, ireq *ingest.ShardedRequest) error {
if uri == nil {
uri = c.defaultURI
}
u := uri.Path(fmt.Sprintf("/internal/ingest/%s/node", indexName))
buf, err := c.serializer.Marshal(ireq)
if err != nil {
return errors.Wrap(err, "marshalling")
}
req, err := http.NewRequest("POST", u, bytes.NewReader(buf))
if err != nil {
return errors.Wrap(err, "creating request")
}
req.Header.Set("Content-Length", strconv.Itoa(len(buf)))
req.Header.Set("Content-Type", "application/x-protobuf")
req.Header.Set("Accept", "application/x-protobuf")
req.Header.Set("User-Agent", "pilosa/"+pilosa.Version)
resp, err := c.executeRequest(req.WithContext(ctx))
if err != nil {
return errors.Wrap(err, "executing request")
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return errors.Errorf("unexpected status code: %s", resp.Status)
}
return nil
}
// MutexCheck uses the mutex-check endpoint to request mutex collision data
// from a single node. It produces per-shard results, and does not translate
// them.

66
http/handler.go
View file

@ -42,6 +42,7 @@ import (
"github.com/gorilla/mux"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/encoding/proto"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/pql"
"github.com/molecula/featurebase/v2/rbf"
@ -432,7 +433,9 @@ func newRouter(handler *Handler) http.Handler {
router.HandleFunc("/internal/index/{index}/shards", handler.handleGetIndexAvailableShards).Methods("GET").Name("GetIndexAvailableShards")
router.HandleFunc("/internal/nodes", handler.handleGetNodes).Methods("GET").Name("GetNodes")
router.HandleFunc("/internal/shards/max", handler.handleGetShardsMax).Methods("GET").Name("GetShardsMax") // TODO: deprecate, but it's being used by the client
router.HandleFunc("/internal/ingest/{index}", handler.handleIngestData).Methods("POST").Name("PostIngestData")
router.HandleFunc("/internal/ingest/{index}", handler.handlePostIngestData).Methods("POST").Name("PostIngestData")
router.HandleFunc("/internal/ingest/{index}/node", handler.handlePostIngestNode).Methods("POST").Name("PostIngestNode")
router.HandleFunc("/internal/schema", handler.handleIngestSchema).Methods("POST").Name("PostIngestSchema")
router.HandleFunc("/internal/translate/index/{index}/keys/find", handler.handleFindIndexKeys).Methods("POST").Name("FindIndexKeys")
router.HandleFunc("/internal/translate/index/{index}/keys/create", handler.handleCreateIndexKeys).Methods("POST").Name("CreateIndexKeys")
@ -524,7 +527,7 @@ func newStatikHandler(h *Handler) statikHandler {
func (s statikHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.UserAgent(), "curl") {
msg := "Welcome. Pilosa v" + s.handler.api.Version() + " is running. Visit https://www.pilosa.com/docs/ for more information."
msg := "Welcome. FeatureBase v" + s.handler.api.Version() + " is running. Visit https://docs.molecula.cloud for more information."
if s.statikFS != nil {
msg += " Try the Web UI by visiting this URL in your browser."
}
@ -1337,7 +1340,9 @@ func (h *Handler) handlePostField(w http.ResponseWriter, r *http.Request) {
resp.write(w, err)
}
func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) {
// handlePostIngestData handles JSON ingest data that may need key
// translation, for the entire cluster.
func (h *Handler) handlePostIngestData(w http.ResponseWriter, r *http.Request) {
if !validHeaderAcceptJSON(r.Header) {
http.Error(w, "JSON only acceptable response", http.StatusNotAcceptable)
return
@ -1351,6 +1356,15 @@ func (h *Handler) handleIngestData(w http.ResponseWriter, r *http.Request) {
qcx := h.api.Txf().NewQcx()
err := h.api.IngestOperations(r.Context(), qcx, indexName, r.Body)
if err == nil {
err = qcx.Finish()
if err != nil {
http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
return
}
} else {
qcx.Abort()
}
resp := successResponse{h: h, Name: indexName}
resp.write(w, err)
@ -2908,6 +2922,52 @@ func (h *Handler) handlePostImportRoaring(w http.ResponseWriter, r *http.Request
}
}
// handlePostIngestNode is the internal endpoint taking already-translated
// ingest operations, sorted by shard, for a single node.
func (h *Handler) handlePostIngestNode(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if error, code := validateProtobufHeader(r); error != "" {
http.Error(w, error, code)
return
}
ctx := r.Context()
// Read entire body.
span, _ := tracing.StartSpanFromContext(ctx, "ioutil.ReadAll-Body")
body, err := readBody(r)
span.LogKV("bodySize", len(body))
span.Finish()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
req := &ingest.ShardedRequest{}
span, _ = tracing.StartSpanFromContext(ctx, "Unmarshal")
err = proto.DefaultSerializer.Unmarshal(body, req)
span.Finish()
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
urlVars := mux.Vars(r)
indexName := urlVars["index"]
qcx := h.api.Txf().NewQcx()
err = h.api.IngestNodeOperations(r.Context(), qcx, indexName, req)
if err == nil {
err = qcx.Finish()
if err != nil {
http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
}
} else {
http.Error(w, fmt.Sprintf("ingesting: %v", err), http.StatusInternalServerError)
qcx.Abort()
}
}
func (h *Handler) handlePostTranslateKeys(w http.ResponseWriter, r *http.Request) {
// Verify that request is only communicating over protobufs.
if r.Header.Get("Content-Type") != "application/x-protobuf" {

14
idalloc.go
View file

@ -53,18 +53,20 @@ func (k IDAllocKey) String() string {
}
type idAllocator struct {
db *bolt.DB
db *bolt.DB
fsyncEnabled bool
}
type OpenIDAllocatorFunc func(path string) (*idAllocator, error) // whyyyyyyyyy
type OpenIDAllocatorFunc func(path string, enableFsync bool) (*idAllocator, error) // whyyyyyyyyy
func OpenIDAllocator(path string) (*idAllocator, error) {
db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second})
func OpenIDAllocator(path string, enableFsync bool) (*idAllocator, error) {
db, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !enableFsync})
if err != nil {
return nil, err
}
return &idAllocator{db}, nil
return &idAllocator{db: db, fsyncEnabled: enableFsync}, nil
}
func (ida *idAllocator) Replace(reader io.Reader) error {
newFile := ida.db.Path() + ".bak"
liveFile := ida.db.Path()
@ -92,7 +94,7 @@ func (ida *idAllocator) Replace(reader io.Reader) error {
} else {
_ = os.Remove(liveFile + ".sav")
}
db, err := bolt.Open(liveFile, 0666, &bolt.Options{Timeout: 1 * time.Second})
db, err := bolt.Open(liveFile, 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: !ida.fsyncEnabled})
ida.db = db
return err
}

2
idalloc_test.go
View file

@ -40,7 +40,7 @@ func TestIDAlloc(t *testing.T) {
}()
// Open bolt.
db, err := bolt.Open(f.Name(), 0666, &bolt.Options{Timeout: 1 * time.Second})
db, err := bolt.Open(f.Name(), 0666, &bolt.Options{Timeout: 1 * time.Second, NoSync: true})
if err != nil {
t.Errorf("opening bolt: %v", err)
return

2
index.go
View file

@ -249,7 +249,7 @@ func (i *Index) open(idx *disco.Index) (err error) {
partitionID := partitionID
g.Go(func() error {
store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN)
store, err := i.OpenTranslateStore(i.TranslateStorePath(partitionID), i.name, "", partitionID, i.holder.partitionN, i.holder.cfg.StorageConfig.FsyncEnabled)
if err != nil {
return errors.Wrapf(err, "opening index translate store: partition=%d", partitionID)
}

2
index_internal_test.go
View file

@ -26,7 +26,7 @@ func mustOpenIndex(tb testing.TB, opt IndexOptions) *Index {
if err != nil {
panic(err)
}
h := NewHolder(path, nil)
h := NewHolder(path, mustHolderConfig())
index, err := h.CreateIndex("i", opt)
testhook.Cleanup(tb, func() {
h.Close()

2
index_test.go
View file

@ -261,7 +261,7 @@ func TestIndex_InvalidName(t *testing.T) {
if err != nil {
panic(err)
}
index, err := pilosa.NewIndex(pilosa.NewHolder(path, nil), path, "ABC")
index, err := pilosa.NewIndex(pilosa.NewHolder(path, mustHolderConfig()), path, "ABC")
if err == nil {
t.Fatalf("should have gotten an error on index name with caps")
}

746
ingest/codec.go
View file

@ -15,10 +15,13 @@
package ingest
import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"math"
"sort"
"strconv"
"time"
@ -37,15 +40,22 @@ import (
// decimal yes yes no
// timestamp yes yes no
type KeyLookupFunc func(...string) (map[string]uint64, error)
// KeyTranslator is a thing that can translate strings to IDs, and also
// IDs back to strings. The ID->string conversion is used only to render
// a request back to JSON, which in turn is only used in testing. The
// functions optionally take an existing map which they then augment.
type KeyTranslator interface {
TranslateKeys(keys ...string) (map[string]uint64, error)
TranslateIDs(ids ...uint64) (map[uint64]string, error)
}
// Codec is a single-use parser which decodes data into columnar vectors.
type Codec interface {
AddSetField(name string, lookup KeyLookupFunc) error
AddTimeQuantumField(name string, lookup KeyLookupFunc) error
AddMutexField(name string, lookup KeyLookupFunc) error
AddSetField(name string, keys KeyTranslator) error
AddTimeQuantumField(name string, keys KeyTranslator) error
AddMutexField(name string, keys KeyTranslator) error
AddBoolField(name string) error
AddIntField(name string, lookup KeyLookupFunc) error
AddIntField(name string, keys KeyTranslator) error
AddDecimalField(name string, scale int64) error
AddTimestampField(name string, scale time.Duration, epoch int64) error
@ -56,138 +66,299 @@ type Codec interface {
type jsonDecFn func(recID uint64, typ jsonparser.ValueType, data []byte) error
// applyTranslationFn is a function which applies key lookups to the values
// of an operation, meaning it needs to know whether it's applying them
// to the signed or unsigned values.
type applyTranslationFn func(*FieldOperation, []uint64) error
// jsonEncFn encodes a value, or range of values, according to a given
// fieldCodec's translation rules. key values, if present, will be used to
// replace whichever values could have been provided as keys. so for instance,
// with an int field which is keyed, values are actually of type int64, but
// if strings are present, those are used. for a time quantum field, the time
// is set from the signed field, and the value is replaced by the key if
// keys are provided.
type jsonEncFn func(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error
type jsonFieldCodec struct {
// fieldCodec represents something that can encode and decode a
// particular field. Its methods are not reentrant, it uses internal
// buffers.
type fieldCodec struct {
valueKeys *StringTable
currentOp *FieldOperation
decode jsonDecFn
translate applyTranslationFn
encode jsonEncFn
// For timestamp: scale-in-nanoseconds; for instance, if scaleUnit is
// 1,000,000,000, we are storing numbers-of-seconds since the Unix epoch.
// The actual value recorded in BSI will be offset by the field's
// epoch, but we don't need to know that.
// For decimal: Decimal digits of precision. So for instance, with
// scaleUnit 2, "1" is stored as 100 and "1.2" is stored as 120.
// scale 2, scaleUnit is 100, "1" is stored as 100 and "1.2" is stored as
// 120.
scaleUnit int64
scale int64
epoch int64 // used only by Timestamp fields
scratch []uint64 // reusable scratch space for sets of values
lookup KeyLookupFunc
keys KeyTranslator
buf []byte // scratch space for format operations.
fieldType FieldType
}
// JSONCodec is a Codec which accepts a JSON map of record keys/ids to updated value maps.
type JSONCodec struct {
recKeys *StringTable
fields map[string]*jsonFieldCodec
keyLookup KeyLookupFunc
currentOp *Operation
fieldTypes map[string]FieldType
recKeys *StringTable
fields map[string]*fieldCodec
keys KeyTranslator
currentOp *Operation
}
// jsonBuffer is a bytes.Buffer which has an associated json.Encoder which
// can be used to write to its buffer. Both types are just embedded because
// they have non-overlapping APIs. Please don't look at my horrible face.
//
// This has some internal objects it can use to stash things
// so that it can pass &foo to enc.Encode() without needing to heap-allocate
// something for the interface conversion, and a buffer it can use for
// converting numbers or times.
//
// It also has an internal error buffer. In fact, in many cases, errors
// are so far as I can tell absolutely impossible -- bytes.Buffer specifically
// promises never to yield an error, and nothing seems to hint that
// enc.Encode can error on integers, strings, booleans, or arrays. So we
// have dozens of error checks that we can't cause to happen even with
// malformed inputs... so we eat those errors, don't require them to be
// checked externally, and return them when done if anyone checks.
type jsonBuffer struct {
*bytes.Buffer
enc *json.Encoder
buf [48]byte // scratch space for using strconv.AppendInt, etc
uintbuf []uint64 // dummy buffer so that we don't have to alloc to print []uint64
strbuf []string // dummy buffer so that we don't have to alloc to print []string
str string // and again, "so we don't have to alloc a copy"
err error
}
// Encode removes the stray newlines added by json.Encoder.
func (j *jsonBuffer) Encode(v interface{}) {
err := j.enc.Encode(v)
if err == nil {
j.Truncate(j.Len() - 1)
} else {
j.err = err
}
}
// EncodeInt uses AppendInt into a static buffer to reduce allocs.
func (j *jsonBuffer) EncodeInt(i int64) {
rep := strconv.AppendInt(j.buf[:0], i, 10)
_, _ = j.Write(rep)
}
// EncodeUint uses AppendUint into a static buffer to reduce allocs.
func (j *jsonBuffer) EncodeUint(u uint64) {
rep := strconv.AppendUint(j.buf[:0], u, 10)
_, _ = j.Write(rep)
}
// EncodeUints uses a static copy of a []uint64 -- the slice, not its
// contents -- in already-allocated memory so the interface conversion
// doesn't have to do that.
func (j *jsonBuffer) EncodeUints(u []uint64) {
j.uintbuf = u
j.Encode(&j.uintbuf)
j.strbuf = nil
}
// EncodeStrings uses a static copy of a []string -- the slice, not its
// contents -- in already-allocated memory so the interface conversion
// doesn't have to do that.
func (j *jsonBuffer) EncodeStrings(s []string) {
j.strbuf = s
j.Encode(&j.strbuf)
j.strbuf = nil
}
// EncodeQuotedUint uses AppendUint into a static buffer to reduce allocs,
// while also surrounding the value with quotes.
func (j *jsonBuffer) EncodeQuotedUint(u uint64) {
j.buf[0] = '"'
rep := strconv.AppendUint(j.buf[1:1], u, 10)
j.buf[len(rep)+1] = '"'
_, _ = j.Write(j.buf[:len(rep)+2])
}
// EncodeString appends the JSON encoding of a string. This exists
// because otherwise runtime allocates a heap-allocated copy of the
// string to live inside an interface{} for the duration of a function
// call...
func (j *jsonBuffer) EncodeString(s string) {
j.str = s
j.Encode(&j.str)
j.str = ""
}
// EncodeTime exists because time's MarshalJSON allocates a new
// buffer every time it gets called, resulting in a full 13% of
// all the allocations produced in a test run, plus another 13%
// or so of them which were for the copies of the time objects
// made to stuff them into an interface{}. Eww.
func (j *jsonBuffer) EncodeTime(t time.Time) {
j.buf[0] = '"'
rep := t.AppendFormat(j.buf[1:1], time.RFC3339Nano)
j.buf[len(rep)+1] = '"'
_, _ = j.Write(j.buf[:len(rep)+2])
}
// EncodeBool writes a literal representation directly to reduce allocs.
func (j *jsonBuffer) EncodeBool(b bool) {
if b {
_, _ = j.WriteString("true")
} else {
_, _ = j.WriteString("false")
}
}
func (j *jsonBuffer) Err() error {
return j.err
}
func newJSONBuffer(data []byte) *jsonBuffer {
e := &jsonBuffer{Buffer: bytes.NewBuffer(data)}
e.enc = json.NewEncoder(e.Buffer)
e.enc.SetEscapeHTML(false) // we are not doing HTML, just JSON
return e
}
var _ Codec = &JSONCodec{}
func NewJSONCodec(lookup KeyLookupFunc) (*JSONCodec, error) {
j := &JSONCodec{fields: map[string]*jsonFieldCodec{}}
if lookup != nil {
func NewJSONCodec(keys KeyTranslator) (*JSONCodec, error) {
j := &JSONCodec{
fields: map[string]*fieldCodec{},
fieldTypes: map[string]FieldType{},
}
if keys != nil {
j.recKeys = NewStringTable()
j.keyLookup = lookup
j.keys = keys
}
return j, nil
}
func (codec *JSONCodec) AddTimeQuantumField(name string, lookup KeyLookupFunc) error {
fieldCodec := &jsonFieldCodec{}
func (codec *JSONCodec) AddTimeQuantumField(name string, keys KeyTranslator) error {
fieldCodec := &fieldCodec{
fieldType: FieldTypeTimeQuantum,
keys: keys,
}
fieldCodec.decode = fieldCodec.DecodeTimeQuantumValue
if lookup != nil {
fieldCodec.translate = (*FieldOperation).TranslateUnsigned
}
return codec.addField(name, FieldTypeTimeQuantum, fieldCodec, lookup)
fieldCodec.encode = fieldCodec.EncodeTimeQuantumValue
return codec.addField(name, fieldCodec)
}
func (codec *JSONCodec) AddSetField(name string, lookup KeyLookupFunc) error {
fieldCodec := &jsonFieldCodec{}
func (codec *JSONCodec) AddSetField(name string, keys KeyTranslator) error {
fieldCodec := &fieldCodec{
fieldType: FieldTypeSet,
keys: keys,
}
fieldCodec.decode = fieldCodec.DecodeSetValue
if lookup != nil {
fieldCodec.translate = (*FieldOperation).TranslateUnsigned
}
return codec.addField(name, FieldTypeSet, fieldCodec, lookup)
fieldCodec.encode = fieldCodec.EncodeSetValue
return codec.addField(name, fieldCodec)
}
func (codec *JSONCodec) AddIntField(name string, lookup KeyLookupFunc) error {
fieldCodec := &jsonFieldCodec{}
func (codec *JSONCodec) AddIntField(name string, keys KeyTranslator) error {
fieldCodec := &fieldCodec{
fieldType: FieldTypeInt,
keys: keys,
}
fieldCodec.decode = fieldCodec.DecodeIntValue
if lookup != nil {
fieldCodec.translate = (*FieldOperation).TranslateSigned
}
return codec.addField(name, FieldTypeInt, fieldCodec, lookup)
fieldCodec.encode = fieldCodec.EncodeIntValue
return codec.addField(name, fieldCodec)
}
func (codec *JSONCodec) AddMutexField(name string, lookup KeyLookupFunc) error {
fieldCodec := &jsonFieldCodec{}
fieldCodec.decode = fieldCodec.DecodeMutexValue
if lookup != nil {
fieldCodec.translate = (*FieldOperation).TranslateUnsigned
func (codec *JSONCodec) AddMutexField(name string, keys KeyTranslator) error {
fieldCodec := &fieldCodec{
fieldType: FieldTypeMutex,
keys: keys,
}
return codec.addField(name, FieldTypeMutex, fieldCodec, lookup)
fieldCodec.decode = fieldCodec.DecodeMutexValue
fieldCodec.encode = fieldCodec.EncodeMutexValue
return codec.addField(name, fieldCodec)
}
func (codec *JSONCodec) AddBoolField(name string) error {
fieldCodec := &jsonFieldCodec{}
fieldCodec := &fieldCodec{
fieldType: FieldTypeBool,
}
fieldCodec.decode = fieldCodec.DecodeBoolValue
return codec.addField(name, FieldTypeBool, fieldCodec, nil)
fieldCodec.encode = fieldCodec.EncodeBoolValue
return codec.addField(name, fieldCodec)
}
// TimestampField is used to store seconds since unix epoch. The numeric values
// stored are adjusted based on the given time.Duration; for instance, if the
// time scale is time.Second, then the second after the epoch is stored as 1,
// if it's time.Millisecond, then it's stored as 1000, etcetera.
// if it's time.Millisecond, then it's stored as 1000, etcetera. The epoch
// passed to this function should be the offset from the Unix epoch to the
// desired epoch, in the same scale. (So if the scale is milliseconds,
// it should be the Unix timestamp in seconds, times 1000.)
func (codec *JSONCodec) AddTimestampField(name string, timeScale time.Duration, epoch int64) error {
fieldCodec := &jsonFieldCodec{scaleUnit: int64(timeScale), epoch: epoch}
fieldCodec := &fieldCodec{
fieldType: FieldTypeTimeStamp,
scaleUnit: int64(timeScale),
epoch: epoch,
}
fieldCodec.decode = fieldCodec.DecodeTimeValue
return codec.addField(name, FieldTypeTimeStamp, fieldCodec, nil)
fieldCodec.encode = fieldCodec.EncodeTimeValue
return codec.addField(name, fieldCodec)
}
// AddDecimalField adds a decimal field, which is stored as integer values
// with a scale offset, but parsed as floating point values. For instance,
// with decimalScale=2, `0.01` would store the value 1.
func (codec *JSONCodec) AddDecimalField(name string, decimalScale int64) error {
fieldCodec := &jsonFieldCodec{scaleUnit: int64(math.Pow10(int(decimalScale)))}
fieldCodec := &fieldCodec{
fieldType: FieldTypeDecimal,
scale: decimalScale,
scaleUnit: int64(math.Pow(10, float64(decimalScale))),
}
fieldCodec.decode = fieldCodec.DecodeDecimalValue
return codec.addField(name, FieldTypeDecimal, fieldCodec, nil)
fieldCodec.encode = fieldCodec.EncodeDecimalValue
return codec.addField(name, fieldCodec)
}
func (codec *JSONCodec) addField(name string, fieldType FieldType, fieldCodec *jsonFieldCodec, lookup KeyLookupFunc) error {
func (codec *JSONCodec) addField(name string, fieldCodec *fieldCodec) error {
if _, ok := codec.fields[name]; ok {
return fmt.Errorf("duplicate field %q", name)
}
if lookup != nil {
if fieldCodec.keys != nil {
fieldCodec.valueKeys = NewStringTable()
fieldCodec.lookup = lookup
}
fieldCodec.fieldType = fieldType
codec.fields[name] = fieldCodec
codec.fieldTypes[name] = fieldCodec.fieldType
return nil
}
// decodeSetOrValue decodes a value which might be either an array of values or a
// single value, where values might be string keys or bare numbers, calling cb for
// each value it finds.
func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) {
func (j *fieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []byte, cb func(uint64) error) (err error) {
switch dataType {
case jsonparser.Array:
_, arrayErr := jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, unused error) {
id, idErr := j.valueKeys.ID(value)
if idErr != nil {
err = idErr
return
}
// stash an error if we got one
valueErr := cb(id)
if valueErr != nil {
err = valueErr
var id uint64
switch dataType {
case jsonparser.String:
id, err = j.valueKeys.ID(value)
if err != nil {
return
}
err = cb(id)
case jsonparser.Number:
if j.valueKeys != nil {
err = errors.New("expecting key, got numeric value")
return
}
id, err = strconv.ParseUint(pretendByteIsString(value), 10, 64)
if err != nil {
return
}
err = cb(id)
default:
err = fmt.Errorf("expecting value or array, got %v", dataType)
}
})
if err != nil {
@ -212,23 +383,32 @@ func (j *jsonFieldCodec) decodeSetOrValue(dataType jsonparser.ValueType, data []
}
return cb(id)
default:
return fmt.Errorf("expecting array, got %v", dataType)
return fmt.Errorf("expecting value or array, got %v", dataType)
}
return err
}
// DecodeSetValue decodes a set of unsigned values from the provided data into the
// associated currentOp.
func (j *jsonFieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) {
func (j *fieldCodec) DecodeSetValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) {
return j.decodeSetOrValue(dataType, data, func(id uint64) error {
j.currentOp.AddPair(recID, id)
return nil
})
}
func (j *fieldCodec) EncodeSetValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
if len(keys) == 1 || len(values) == 1 {
// simplify: just hand this off to a single-value case
return j.EncodeMutexValue(dst, values, signed, keys)
}
appendKeysJSON(dst, values, keys)
return nil
}
// DecodeIntValue decodes a single signed value from the provided data
// into the associated currentOp.
func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
func (j *fieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
switch dataType {
case jsonparser.String:
value, err := j.valueKeys.IntID(data)
@ -255,25 +435,62 @@ func (j *jsonFieldCodec) DecodeIntValue(recID uint64, dataType jsonparser.ValueT
return nil
}
func (j *fieldCodec) EncodeIntValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
if len(keys) == 0 {
if len(signed) == 0 {
return errors.New("encodeIntValue: need a value")
}
dst.EncodeInt(signed[0])
return nil
}
dst.EncodeString(keys[0])
return nil
}
// DecodeMutexValue decodes a single unsigned value from the provided data
// into the associated currentOp.
func (j *jsonFieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
func (j *fieldCodec) DecodeMutexValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
switch dataType {
case jsonparser.Number, jsonparser.String:
id, err := j.valueKeys.ID(data)
case jsonparser.String:
value, err := j.valueKeys.ID(data)
if err != nil {
return err
}
j.currentOp.AddPair(recID, id)
j.currentOp.AddPair(recID, value)
case jsonparser.Number:
if j.valueKeys != nil {
return errors.New("expecting string key, got numeric value")
}
value, err := strconv.ParseUint(pretendByteIsString(data), 10, 64)
if err != nil {
return err
}
j.currentOp.AddPair(recID, value)
default:
return fmt.Errorf("expecting integer value, got %v", dataType)
if j.valueKeys != nil {
return fmt.Errorf("expecting string key, got %v", dataType)
} else {
return fmt.Errorf("expecting integer value, got %v", dataType)
}
}
return nil
}
func (j *fieldCodec) EncodeMutexValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
if len(keys) == 0 {
if len(values) == 0 {
return errors.New("encodeMutexValue: need a value")
}
dst.EncodeUint(values[0])
return nil
}
dst.EncodeString(keys[0])
return nil
}
// DecodeBoolValue decodes a single true/false value from the provided data
// into the associated currentOp.
func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error {
func (j *fieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType, data []byte) error {
value := uint64(0)
switch typ {
case jsonparser.String:
@ -303,9 +520,21 @@ func (j *jsonFieldCodec) DecodeBoolValue(recID uint64, typ jsonparser.ValueType,
return nil
}
func (j *fieldCodec) EncodeBoolValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
if len(values) == 0 {
return errors.New("encoding boolean value, but none provided")
}
var x bool
if values[0] != 0 {
x = true
}
dst.EncodeBool(x)
return nil
}
// DecodeTimeQuantumValue decodes a timestamp, and a set of bits from the
// provided data into the associated currentOp.
func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error {
func (j *fieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.ValueType, data []byte) error {
j.scratch = j.scratch[:0]
stamp := time.Unix(0, 0).UTC()
err := jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) (err error) {
@ -346,8 +575,27 @@ func (j *jsonFieldCodec) DecodeTimeQuantumValue(recID uint64, typ jsonparser.Val
return nil
}
// Our format does not allow a record to have multiple values set with
// different timestamps at the same time. We use the first timestamp for
// the whole set.
func (j *fieldCodec) EncodeTimeQuantumValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
if len(values) == 0 {
return errors.New("encoding time quantum value: no value provided")
}
dst.WriteString(`{"values":`)
appendKeysJSON(dst, values, keys)
// a zero timestamp is idiomatic for no-time-provided, and i sort of
// hate that, but here we are.
if len(signed) > 0 && signed[0] != 0 {
_, _ = dst.WriteString(`,"time":`)
dst.EncodeTime(time.Unix(0, signed[0]).UTC())
}
_, _ = dst.WriteString(`}`)
return nil
}
// DecodeTimeValue will eventually work but right now it doesn't actually.
func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) {
func (j *fieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.ValueType, data []byte) (err error) {
var stamp time.Time
switch dataType {
case jsonparser.String:
@ -364,12 +612,22 @@ func (j *jsonFieldCodec) DecodeTimeValue(recID uint64, dataType jsonparser.Value
return fmt.Errorf("parsing numeric timestamp: %w", err)
}
j.currentOp.AddSignedPair(recID, i64)
default:
return fmt.Errorf("expecting time, got %s", dataType)
}
return nil
}
func (j *fieldCodec) EncodeTimeValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
if len(signed) == 0 {
return errors.New("encoding time value: no value provided")
}
dst.EncodeTime(time.Unix(0, (signed[0]+j.epoch)*j.scaleUnit).UTC())
return nil
}
// DecodeDecimalValue will eventually work but right now it doesn't actually.
func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
func (j *fieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.ValueType, data []byte) error {
switch dataType {
case jsonparser.String, jsonparser.Number:
value, err := jsonparser.GetFloat(data)
@ -384,6 +642,28 @@ func (j *jsonFieldCodec) DecodeDecimalValue(recID uint64, dataType jsonparser.Va
return nil
}
func (j *fieldCodec) EncodeDecimalValue(dst *jsonBuffer, values []uint64, signed []int64, keys []string) error {
j.buf = strconv.AppendInt(j.buf[:0], signed[0], 10)
scale := int(j.scale)
if len(j.buf) > scale {
j.buf = append(j.buf, '.')
// shove the last scaleUnit values over
copy(j.buf[len(j.buf)-scale:], j.buf[len(j.buf)-scale-1:])
j.buf[len(j.buf)-scale-1] = '.'
}
_, _ = dst.Write(j.buf)
return nil
}
// FieldTypes gives a mapping of fields to their basic types used by this
// codec.
func (codec *JSONCodec) FieldTypes() map[string]FieldType {
return codec.fieldTypes
}
// ParseKeyedRecords parses the records it finds. Note that, when you're doing
// a Write op, this will update ClearRecordIDs automatically as it goes,
// even though record_ids isn't specified in the JSON for that case.
func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) {
seen := make(map[uint64]struct{})
return jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
@ -412,8 +692,8 @@ func (codec *JSONCodec) ParseKeyedRecords(data []byte) (err error) {
})
}
func (codec *JSONCodec) ParseOperation(data []byte) (op *Operation, err error) {
op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields))}
func (codec *JSONCodec) ParseOperation(data []byte, seq int) (op *Operation, err error) {
op = &Operation{FieldOps: make(map[string]*FieldOperation, len(codec.fields)), Seq: seq}
codec.currentOp = op
err = jsonparser.ObjectEach(data, func(key []byte, value []byte, dataType jsonparser.ValueType, offset int) error {
switch string(key) {
@ -491,10 +771,12 @@ func (codec *JSONCodec) Parse(r io.Reader) (req *Request, err error) {
func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) {
var ops []*Operation
var lastErr error
var seq int
_, err = jsonparser.ArrayEach(data, func(value []byte, dataType jsonparser.ValueType, offset int, err error) {
switch dataType {
case jsonparser.Object:
op, err := codec.ParseOperation(value)
op, err := codec.ParseOperation(value, seq)
seq++
if err != nil {
lastErr = fmt.Errorf("parsing operation: %v", err)
return
@ -512,34 +794,39 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) {
}
// and now, key translation!
var keyMap []uint64
if codec.keyLookup != nil {
keyMap, err = MapForStringTable(codec.recKeys, codec.keyLookup)
if codec.keys != nil {
keyMap, err = codec.recKeys.MakeIDMap(codec.keys)
if err != nil {
return nil, fmt.Errorf("trying to find record key mapping: %w", err)
}
}
req = &Request{FieldTypes: make(map[string]FieldType, len(codec.fields))}
req = &Request{}
valueMaps := map[string]func(*FieldOperation) error{}
for name, fieldCodec := range codec.fields {
// make closure survive iteration
fieldCodec := fieldCodec
if fieldCodec.lookup != nil {
fieldMap, err := MapForStringTable(fieldCodec.valueKeys, fieldCodec.lookup)
if fieldCodec.keys != nil {
fieldMap, err := fieldCodec.valueKeys.MakeIDMap(fieldCodec.keys)
if err != nil {
return nil, fmt.Errorf("trying to find value mapping for %q: %w", name, err)
}
valueMaps[name] = func(fo *FieldOperation) error {
return fieldCodec.translate(fo, fieldMap)
if fieldCodec.fieldType == FieldTypeInt {
valueMaps[name] = func(fo *FieldOperation) error {
return translateSigned(fieldMap, fo.Signed)
}
} else {
valueMaps[name] = func(fo *FieldOperation) error {
return translateUnsigned(fieldMap, fo.Values)
}
}
}
req.FieldTypes[name] = fieldCodec.fieldType
}
for _, op := range ops {
// For Clear and Write, we need to translate/sort our record ID
// list.
if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete {
if keyMap != nil {
if err = translateUnsignedSlice(op.ClearRecordIDs, keyMap); err != nil {
if err = translateUnsigned(keyMap, op.ClearRecordIDs); err != nil {
return nil, fmt.Errorf("mapping record keys for clear op: %w", err)
}
}
@ -554,7 +841,7 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) {
continue
}
if keyMap != nil {
if err = fieldOp.TranslateKeys(keyMap); err != nil {
if err = translateUnsigned(keyMap, fieldOp.RecordIDs); err != nil {
return nil, fmt.Errorf("mapping record keys for op on %q: %w", field, err)
}
}
@ -574,6 +861,283 @@ func (codec *JSONCodec) ParseBytes(data []byte) (req *Request, err error) {
return req, nil
}
// AppendBytes appends bytes which we would expect to produce the same
// request, using this codec. It does not try to recreate FieldTypes, which
// would be handled by the codec anyway. It's written as an Append so you
// can reuse a buffer.
func (codec *JSONCodec) AppendBytes(req *Request, data []byte) (out []byte, err error) {
if req == nil || len(req.Ops) == 0 {
return append(data, "[]"...), nil
}
dst := newJSONBuffer(data)
// we ignore the FieldTypes part of the Request, which is just there to
// let the request's ByShard use the correct sorting routines, which is
// itself sort of awful. The codec will insert it again when parsing the
// bytes.
_, _ = dst.WriteString(`,`)
for _, op := range req.Ops {
// EncodeJSON needs to have access to this codec's field data
// and key translators.
err = op.EncodeJSON(dst, codec)
if err != nil {
return nil, err
}
_, _ = dst.WriteString(`,`)
}
// delete the last comma
dst.Truncate(dst.Len() - 1)
_, _ = dst.WriteString(`]`)
// there's very few ways an error could occur, possibly none, but
// just in case lots of internal operations stashed an error if one
// happened, so we'll return anything they came up with.
return dst.Bytes(), dst.Err()
}
// appendIDsJSON appends the provided IDs to a JSON stream as a bracketed
// list of quoted strings if there's a key translator, or integer values
// if the KeyTranslator is nil, comma-separated. It can error because a
// translator can error.
func (codec *JSONCodec) appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error {
return appendIDsJSON(dst, values, keys)
}
func appendKeysJSON(dst *jsonBuffer, values []uint64, keys []string) {
if len(values) == 0 && len(keys) == 0 {
_, _ = dst.WriteString("[]")
return
}
if len(keys) != 0 {
dst.EncodeStrings(keys)
} else {
dst.EncodeUints(values)
}
}
func appendIDsJSON(dst *jsonBuffer, values []uint64, keys KeyTranslator) error {
if len(values) == 0 {
_, _ = dst.WriteString("[]")
return nil
}
if keys == nil {
dst.EncodeUints(values)
return nil
}
_, _ = dst.WriteString(`[`)
translated, err := keys.TranslateIDs(values...)
if err != nil {
return err
}
for _, v := range values {
dst.EncodeString(translated[v])
_, _ = dst.WriteString(`,`)
}
dst.Truncate(dst.Len() - 1)
_, _ = dst.WriteString(`]`)
return nil
}
// RequestByShard makes up for the fact that we don't want to stash the
// field type data in the request, but we need it to actually do the by-shard.
func (codec *JSONCodec) RequestByShard(req *Request) (*ShardedRequest, error) {
return req.ByShard(codec.fieldTypes)
}
// EncodeJSON encodes an operation as JSON using a provided buffer.
// It uses the provided codec where necessary to help with key
// translation.
func (o *Operation) EncodeJSON(dst *jsonBuffer, codec *JSONCodec) (err error) {
// We're not trying to guarantee that what we produce makes sense or is
// identical to what produced us, just to write our current state out.
if o == nil {
_, _ = dst.WriteString(`{}`)
return nil
}
_, _ = dst.WriteString(`{"action":`)
dst.EncodeString(o.OpType.String())
// it is intentional that o.Seq isn't encoded here; it makes no sense
// to allow an op to specify its seq in JSON.
if o.OpType != OpWrite {
// for Write, ClearRecordIDs and ClearFields were computed
// from the records being written, and aren't actually part of the data.
if len(o.ClearRecordIDs) != 0 {
_, _ = dst.WriteString(`,"record_ids":`)
err = codec.appendIDsJSON(dst, o.ClearRecordIDs, codec.keys)
if err != nil {
return err
}
}
if len(o.ClearFields) != 0 {
_, _ = dst.WriteString(`,"fields":`)
dst.EncodeStrings(o.ClearFields)
}
}
if len(o.FieldOps) == 0 {
_, _ = dst.WriteString(`}`)
return
}
// collect all the fields that have a non-empty set of records. we can
// just ignore the others.
fieldNames := make([]string, 0, len(o.FieldOps))
for field, op := range o.FieldOps {
if op != nil && len(op.RecordIDs) != 0 {
fieldNames = append(fieldNames, field)
}
}
// nevermind then
if len(fieldNames) == 0 {
_, _ = dst.WriteString(`}`)
return nil
}
_, _ = dst.WriteString(`,"records":{`)
// now we have to invert the logic, creating records from
// fields with corresponding ops. uh-oh.
fieldOps := make([]*FieldOperation, len(o.FieldOps))
fieldCodecs := make([]*fieldCodec, len(o.FieldOps))
fieldKeys := make([][]string, len(o.FieldOps)) // translated field keys
indexes := make([]int, len(o.FieldOps))
sort.Slice(fieldNames, func(i, j int) bool { return fieldNames[i] < fieldNames[j] })
next := ^uint64(0)
var idKeys map[uint64]string
if codec.keys != nil {
idKeys = make(map[uint64]string)
}
for i, field := range fieldNames {
// populate a parallel slice of field ops so we don't have
// to do map lookups for every single piece of data
op := o.FieldOps[field]
fieldOps[i] = op
fc := codec.fields[field]
if fc == nil {
return fmt.Errorf("unknown field: %q", field)
}
fieldCodecs[i] = fc
if codec.keys != nil {
// we'll build a list of keys we need for any records
for _, v := range op.RecordIDs {
idKeys[v] = ""
}
}
id := op.RecordIDs[0]
if id < next {
next = id
}
if fc.keys != nil {
thisFieldKeys := make([]string, len(op.RecordIDs))
if fc.fieldType == FieldTypeInt {
u := make([]uint64, len(op.Signed))
for i := range op.Signed {
u[i] = uint64(op.Signed[i])
}
valueKeys, err := fc.keys.TranslateIDs(u...)
if err != nil {
return err
}
for j, v := range op.Signed {
thisFieldKeys[j] = valueKeys[uint64(v)]
}
} else {
valueKeys, err := fc.keys.TranslateIDs(op.Values...)
if err != nil {
return err
}
for j, v := range op.Values {
thisFieldKeys[j] = valueKeys[v]
}
}
fieldKeys[i] = thisFieldKeys
}
}
// ... no fieldops actually have any entries. therefore no record will
// have any fields set, therefore no records exist...
if next == ^uint64(0) {
_, _ = dst.WriteString(`}}`)
return nil
}
if codec.keys != nil {
idList := make([]uint64, 0, len(idKeys))
for k := range idKeys {
idList = append(idList, k)
}
idKeys, err = codec.keys.TranslateIDs(idList...)
if err != nil {
return err
}
}
// next is the ID of a record which exists for at least one field.
// we'll recompute it every loop.
for next != ^uint64(0) {
if codec.keys != nil {
dst.EncodeString(idKeys[next])
} else {
dst.EncodeQuotedUint(next)
}
_, _ = dst.WriteString(`:{`)
current := next
next = ^uint64(0)
for i, field := range fieldNames {
op := fieldOps[i]
idx := indexes[i]
if idx >= len(op.RecordIDs) {
continue
}
id := op.RecordIDs[idx]
if id == current {
var j int
// count ahead to first index which is either outside the list
// or a different id
for j = idx; j < len(op.RecordIDs) && op.RecordIDs[j] == id; j++ {
}
// fmt.Printf("field %s encoding %d-%d (v %d, s %d, k %d)\n",
// field, idx, j, len(op.Values), len(op.Signed), len(fieldKeys[i]))
// print this one, and advance this index to next position
dst.EncodeString(field)
_, _ = dst.WriteString(`:`)
var values []uint64
var signed []int64
var keys []string
if len(op.Values) >= j {
values = op.Values[idx:j]
}
if len(op.Signed) >= j {
signed = op.Signed[idx:j]
}
if len(fieldKeys[i]) >= j {
keys = fieldKeys[i][idx:j]
}
err = fieldCodecs[i].encode(dst, values, signed, keys)
if err != nil {
return err
}
_, _ = dst.WriteString(`,`)
indexes[i] = j
// and we'll fall through to the id < next check, so we
// just set id here.
if indexes[i] < len(op.RecordIDs) {
id = op.RecordIDs[indexes[i]]
} else {
id = ^uint64(0)
}
}
if id < next {
next = id
}
}
// we should always have a trailing comma after any entry, and
// if there were no entries we shouldn't have had a list...
dst.Truncate(dst.Len() - 1)
_, _ = dst.WriteString(`},`)
}
dst.Truncate(dst.Len() - 1)
// close both the records list, and the whole object.
_, _ = dst.WriteString(`}}`)
// In theory, there's no way for most of these to produce errors,
// but just in case, we'll check for an error every operation or so.
return dst.Err()
}
type errFieldNotFound struct {
field string
}

990
ingest/codec_test.go
View file

File diff suppressed because it is too large Load diff

345
ingest/op.go
View file

@ -15,6 +15,7 @@
package ingest
import (
"errors"
"fmt"
"math/bits"
"sort"
@ -92,14 +93,22 @@ func ParseOpType(s string) (OpType, error) {
// record IDs and fields, a Set or Remove will have only FieldOps,
// and a Write will have both -- populating the record IDs and fields
// from the fieldops.
//
// When we parse an operation, we assign each op a sequential ID
// within the overall request. We keep these IDs associated with ops
// when splitting them up across shards, so we can reverse this even
// if some shards don't get some ops.
type Operation struct {
OpType OpType
Seq int // sequence position within a chain of ops
ClearRecordIDs []uint64
ClearFields []string
FieldOps map[string]*FieldOperation
}
// Compare reports whether two operations seem to be the same.
// Compare reports whether two Operations seem to be the same. While
// a FieldOperation can be "empty" and compare-equal-to a nil FieldOperation,
// no Operation is considered empty even if it has no FieldOps.
func (got *Operation) Compare(expected *Operation) error {
if got == nil && expected == nil {
return nil
@ -122,27 +131,39 @@ func (got *Operation) Compare(expected *Operation) error {
return fmt.Errorf("clear record id %d differs: expected %d, got %d", i, v2, v1)
}
}
if len(got.ClearFields) != len(expected.ClearFields) {
return fmt.Errorf("clear field counts differ: expected %d (%q), got %d (%q)", len(expected.ClearFields), expected.ClearFields, len(got.ClearFields), got.ClearFields)
// don't assume consistent ordering for the fields, because they're
// coming out in arbitrary hash order
seenFields := make(map[string]struct{}, len(expected.ClearFields))
for _, v1 := range expected.ClearFields {
seenFields[v1] = struct{}{}
}
for i, v1 := range got.ClearFields {
v2 := expected.ClearFields[i]
if v1 != v2 {
return fmt.Errorf("clear field %d differs: expected %q, got %q", i, v2, v1)
for _, v2 := range got.ClearFields {
if _, ok := seenFields[v2]; !ok {
return fmt.Errorf("field %q cleared unexpectedly", v2)
}
delete(seenFields, v2)
}
for v1 := range seenFields {
return fmt.Errorf("field %q should be cleared but wasn't", v1)
}
// We check compare even if the op we find on one side is nil, so
// a round-trip test will consider a missing op and an empty op to
// be interchangeable.
for k, fo1 := range got.FieldOps {
fo2 := expected.FieldOps[k]
if err := fo1.Compare(fo2); err != nil {
return fmt.Errorf("field %q mismatch: %w", k, err)
}
}
for k := range expected.FieldOps {
_, ok := got.FieldOps[k]
if !ok {
return fmt.Errorf("expected op for field %q, but none found", k)
for k, fo2 := range expected.FieldOps {
fo1 := got.FieldOps[k]
if err := fo1.Compare(fo2); err != nil {
return fmt.Errorf("field %q mismatch: %w", k, err)
}
}
if expected.Seq != got.Seq {
return fmt.Errorf("sequence mismatch: expected %d, got %d", expected.Seq, got.Seq)
}
return nil
}
@ -172,6 +193,40 @@ func (o *Operation) Sort() {
sort.Strings(o.ClearFields)
}
// clone makes a duplicate of the operation without shared storage
func (o *Operation) clone() *Operation {
o2 := &Operation{
OpType: o.OpType,
Seq: o.Seq,
ClearRecordIDs: append([]uint64{}, o.ClearRecordIDs...),
ClearFields: append([]string{}, o.ClearFields...),
FieldOps: make(map[string]*FieldOperation, len(o.FieldOps)),
}
for k, v := range o.FieldOps {
o2.FieldOps[k] = v.clone()
}
return o2
}
// merge merges the fieldops of the provided operation into this operation.
func (o *Operation) merge(o2 *Operation) {
o.ClearRecordIDs = append(o.ClearRecordIDs, o2.ClearRecordIDs...)
o.ClearFields = append(o.ClearFields, o2.ClearFields...)
for f, op := range o2.FieldOps {
dst := o.FieldOps[f]
if dst == nil {
o.FieldOps[f] = op
continue
}
// otherwise append any contents. dst is a pointer-to, so this
// updates the thing the map points to, we don't have to write back
// into the map.
dst.RecordIDs = append(dst.RecordIDs, op.RecordIDs...)
dst.Values = append(dst.Values, op.Values...)
dst.Signed = append(dst.Signed, op.Signed...)
}
}
type ShardedFieldOperation map[uint64]*FieldOperation
// ByShard() divides the FieldOperation's values up into corresponding chunks
@ -184,22 +239,14 @@ func (f *FieldOperation) ByShard() ShardedFieldOperation {
return f.SortToShards()
}
// ShardInto puts the shards it finds into a target map.
func (f *FieldOperation) ShardInto(target ShardedFieldOperation) {
shards, ends := shardwidth.FindShards(f.RecordIDs)
prev := 0
for i, shard := range shards {
endIndex := ends[i]
subOp := &FieldOperation{RecordIDs: f.RecordIDs[prev:endIndex]}
if len(f.Values) > 0 {
subOp.Values = f.Values[prev:endIndex]
}
if len(f.Signed) > 0 {
subOp.Signed = f.Signed[prev:endIndex]
}
target[shard] = subOp
prev = endIndex
// clone makes a duplicate of the operation without shared storage
func (f *FieldOperation) clone() *FieldOperation {
f2 := &FieldOperation{
RecordIDs: append([]uint64{}, f.RecordIDs...),
Values: append([]uint64{}, f.Values...),
Signed: append([]int64{}, f.Signed...),
}
return f2
}
func ShardIDs(ids []uint64) (out map[uint64][]uint64) {
@ -349,10 +396,22 @@ func (f *FieldOperation) SortByKeys(keys []uint64) {
// doesn't help as much as you'd hope. This beats using stdlib sort by about
// a factor of two for those small N, for larger N we're using the radix sort
// that calls this.
//
// External keys exist only when we are sorting by value, which is to say,
// when we're using row-oriented formats (set, mutex, time quantum).
// For int/decimal/timestamp fields, we're sorting by record only.
// So, if keys is the same as f.RecordIDs, we're looking at an int field
// or equivalent, so Signed exists and Values doesn't exist.
// Otherwise, we might be looking at a time quantum field (both exist)
// or set/mutex (only Values exist).
func simpleSort(f *FieldOperation, keys []uint64) {
if keys != nil {
// keys might actually just point to record IDs, in which case, we don't
// want to shuffle the corresponding RecordIDs too, because that would just
// reverse our swaps. If they're different, we actually need to swap them
// both.
if &keys[0] != &f.RecordIDs[0] {
// sorting by record IDs
if f.Values != nil && f.Signed != nil {
if f.Values != nil && f.Signed != nil { // time quantum field
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
@ -362,7 +421,7 @@ func simpleSort(f *FieldOperation, keys []uint64) {
}
}
} else if f.Values != nil {
} else if f.Values != nil { // set/mutex/bool
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
@ -371,7 +430,7 @@ func simpleSort(f *FieldOperation, keys []uint64) {
f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1]
}
}
} else if f.Signed != nil {
} else if f.Signed != nil { // can't-happen, we think
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
@ -379,7 +438,7 @@ func simpleSort(f *FieldOperation, keys []uint64) {
f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
}
}
} else {
} else { // can't-happen, we think
for i := 1; i < len(keys); i++ {
for j := i; j > 0 && keys[j-1] > keys[j]; j-- {
keys[j-1], keys[j] = keys[j], keys[j-1]
@ -388,7 +447,14 @@ func simpleSort(f *FieldOperation, keys []uint64) {
}
}
} else {
if f.Values != nil && f.Signed != nil {
if f.Values == nil && f.Signed != nil {
for i := 1; i < len(f.RecordIDs); i++ {
for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
}
}
} else if f.Values != nil && f.Signed != nil { // can't happen, we think
for i := 1; i < len(f.RecordIDs); i++ {
for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
@ -396,22 +462,14 @@ func simpleSort(f *FieldOperation, keys []uint64) {
f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
}
}
} else if f.Values != nil {
} else if f.Values != nil { // only happens during testing
for i := 1; i < len(f.RecordIDs); i++ {
for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
f.Values[j-1], f.Values[j] = f.Values[j], f.Values[j-1]
}
}
} else if f.Signed != nil {
for i := 1; i < len(f.RecordIDs); i++ {
for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
f.Signed[j-1], f.Signed[j] = f.Signed[j], f.Signed[j-1]
}
}
} else {
// why do we only have record IDs? I don't know
} else { // should definitely not happen
for i := 1; i < len(f.RecordIDs); i++ {
for j := i; j > 0 && f.RecordIDs[j-1] > f.RecordIDs[j]; j-- {
f.RecordIDs[j-1], f.RecordIDs[j] = f.RecordIDs[j], f.RecordIDs[j-1]
@ -493,12 +551,7 @@ func sortPartialByKeys(f *FieldOperation, keys []uint64, shift int) {
if end-start > 32 {
sortPartialByKeys(&bucketOp, keys[start:end], nextShift)
} else {
// naive stdlib sort
if externalKeys {
simpleSort(&bucketOp, keys[start:end])
} else {
simpleSort(&bucketOp, nil)
}
simpleSort(&bucketOp, keys[start:end])
}
}
}
@ -534,16 +587,15 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error {
if expected == nil {
return nil
}
if len(expected.RecordIDs) == 0 && len(expected.Values) == 0 && len(expected.Signed) == 0 {
// We don't worry about non-empty Values or Signed here, because in theory
// RecordIDs are the Source of Truth as to what's in the op.
if len(expected.RecordIDs) == 0 {
return nil
}
return fmt.Errorf("expected field operation with %d records, got nil", len(expected.RecordIDs))
}
if expected == nil {
if got == nil {
return nil
}
if len(got.RecordIDs) == 0 && len(got.Values) == 0 && len(got.Signed) == 0 {
if len(got.RecordIDs) == 0 {
return nil
}
return fmt.Errorf("expected empty field operation, got %d records", len(got.RecordIDs))
@ -578,52 +630,6 @@ func (got *FieldOperation) Compare(expected *FieldOperation) error {
return nil
}
func translateUnsignedSlice(target []uint64, mapping []uint64) (err error) {
oops := 0
for i, v := range target {
if v >= uint64(len(mapping)) {
oops++
} else {
target[i] = mapping[v]
}
}
if oops > 0 {
return fmt.Errorf("encountered %d out-of-range keys when applying translation mapping", oops)
}
return nil
}
// TranslateUnsigned translates keys according to the provided mapping. This
// is used for sets, mutexes, and time quantums.
func (op *FieldOperation) TranslateKeys(mapping []uint64) error {
return translateUnsignedSlice(op.RecordIDs, mapping)
}
// TranslateUnsigned translates values according to the provided mapping. This
// is used for sets, mutexes, and time quantums.
func (op *FieldOperation) TranslateUnsigned(mapping []uint64) error {
return translateUnsignedSlice(op.Values, mapping)
}
// TranslateSigned translates signed values according to the provided mapping.
// If we're using this, it's because we're in an integer-type field, which
// admits using keys for fields, so all key values are actually non-negative,
// but the field's type still requires values be expressed as signed ints.
func (op *FieldOperation) TranslateSigned(mapping []uint64) error {
oops := 0
for i, v := range op.Signed {
if v >= int64(len(mapping)) {
oops++
} else {
op.Signed[i] = int64(mapping[v])
}
}
if oops > 0 {
return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops)
}
return nil
}
// ShardOperations is a set of Operations associated with a specific shard.
type ShardOperations struct {
Shard uint64
@ -633,19 +639,17 @@ type ShardOperations struct {
// Request is a complete ingest request, which may be any combination
// of operations, which may apply to multiple shards.
type Request struct {
FieldTypes map[string]FieldType
Ops []*Operation
Ops []*Operation
}
// ShardedRequest is an ingest request, split up into individual per-shard
// operations.
type ShardedRequest struct {
FieldTypes map[string]FieldType
Ops map[uint64][]*Operation
Ops map[uint64][]*Operation
}
// ByShard converts a request into the same request, only sharded.
func (r *Request) ByShard() (*ShardedRequest, error) {
func (r *Request) ByShard(fields map[string]FieldType) (*ShardedRequest, error) {
if len(r.Ops) == 0 {
return &ShardedRequest{Ops: nil}, nil
}
@ -662,12 +666,12 @@ func (r *Request) ByShard() (*ShardedRequest, error) {
if op.OpType == OpClear || op.OpType == OpWrite || op.OpType == OpDelete {
sharded := ShardIDs(op.ClearRecordIDs)
for shard, data := range sharded {
shards[shard] = &Operation{OpType: op.OpType, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}}
shards[shard] = &Operation{OpType: op.OpType, Seq: op.Seq, ClearRecordIDs: data, ClearFields: op.ClearFields, FieldOps: map[string]*FieldOperation{}}
}
}
for field, fieldOp := range op.FieldOps {
sharded := fieldOp.ByShard()
sorter := fieldTypeSorts[r.FieldTypes[field]]
sorter := fieldTypeSorts[fields[field]]
if sorter == nil {
sorter = (*FieldOperation).SortByRecords
}
@ -678,7 +682,7 @@ func (r *Request) ByShard() (*ShardedRequest, error) {
if op.OpType == OpWrite {
return nil, fmt.Errorf("write operation has field operation data (%d items) for shard %d, but no clear data", len(data.RecordIDs), shard)
}
shardOp = &Operation{OpType: op.OpType}
shardOp = &Operation{OpType: op.OpType, Seq: op.Seq}
shards[shard] = shardOp
shardOp.FieldOps = map[string]*FieldOperation{field: data}
} else {
@ -697,18 +701,139 @@ func (r *Request) ByShard() (*ShardedRequest, error) {
return &ShardedRequest{Ops: req}, nil
}
// merge combines the components of a sharded request back into a single
// unsharded request, processing shards in numerical order.
func (s *ShardedRequest) merge() *Request {
req := &Request{}
if s == nil || len(s.Ops) == 0 {
return req
}
shards := make([]uint64, 0, len(s.Ops))
for shard := range s.Ops {
shards = append(shards, shard)
}
sort.Slice(shards, func(i, j int) bool { return shards[i] < shards[j] })
for _, shard := range shards {
ops := s.Ops[shard]
for _, op := range ops {
var _ *Operation
if op.Seq >= len(req.Ops) {
// Pad out with nil *Operations to the required length
req.Ops = append(req.Ops, make([]*Operation, op.Seq+1-len(req.Ops))...)
}
if req.Ops[op.Seq] == nil {
req.Ops[op.Seq] = op.clone()
continue
}
req.Ops[op.Seq].merge(op)
}
}
return req
}
func (r *Request) Dump(logf func(string, ...interface{})) {
logf("req: %#v", r)
for _, op := range r.Ops {
logf("op: %#v", op)
if len(op.ClearRecordIDs) > 0 {
logf(" clearRecordIDs: %d", op.ClearRecordIDs)
if len(op.ClearRecordIDs) > 8 {
logf(" clearRecordIDs: %d...+%d", op.ClearRecordIDs[:8], len(op.ClearRecordIDs)-8)
} else {
logf(" clearRecordIDs: %d", op.ClearRecordIDs)
}
}
if len(op.ClearFields) > 0 {
logf(" clearFields: %s", op.ClearFields)
if len(op.ClearFields) > 8 {
logf(" clearFields: %s...+%d", op.ClearFields[:8], len(op.ClearFields)-8)
} else {
logf(" clearFields: %s", op.ClearFields)
}
}
for field, fieldOp := range op.FieldOps {
logf(" field %q: %#v", field, fieldOp)
if fieldOp != nil {
logf(" field %q: op (%d/%d/%d)", field, len(fieldOp.RecordIDs), len(fieldOp.Values), len(fieldOp.Signed))
if len(fieldOp.RecordIDs) > 0 {
if len(fieldOp.RecordIDs) > 8 {
logf(" records %d...+%d", fieldOp.RecordIDs[:8], len(fieldOp.RecordIDs)-8)
} else {
logf(" records %d", fieldOp.RecordIDs)
}
}
} else {
logf(" field %q: nil op", field)
}
}
}
}
func (r *Request) Compare(other *Request) error {
if other == nil {
if r != nil && len(r.Ops) != 0 {
return errors.New("non-empty sharded request can't equal empty/nil sharded request")
}
// empty and nil are allowed
return nil
}
if r == nil {
if other != nil && len(other.Ops) != 0 {
return errors.New("non-empty sharded request can't equal empty/nil sharded request")
}
// empty and nil are allowed
return nil
}
ops := r.Ops
ops2 := other.Ops
if len(ops2) != len(ops) {
return fmt.Errorf("expected %d ops, got %d", len(ops), len(ops2))
}
for i, op := range ops {
if err := op.Compare(ops2[i]); err != nil {
return fmt.Errorf("op %d: %v", i, err)
}
}
return nil
}
// Compare checks whether two ShardedRequest objects represent the same
// data. Empty shards shouldn't have entries in the map in the first place,
// so we don't accept a nil or 0-length slice of ops as equal to the
// shard key not existing, but we do accept nil or empty requests as
// equal to each other.
func (s *ShardedRequest) Compare(other *ShardedRequest) error {
if other == nil {
if s != nil && len(s.Ops) != 0 {
return errors.New("non-empty sharded request can't equal empty/nil sharded request")
}
// empty and nil are allowed
return nil
}
if s == nil {
if other != nil && len(other.Ops) != 0 {
return errors.New("non-empty sharded request can't equal empty/nil sharded request")
}
// empty and nil are allowed
return nil
}
for shard, ops := range s.Ops {
ops2, ok := other.Ops[shard]
if !ok {
return fmt.Errorf("shard %d missing in other", shard)
}
if len(ops2) != len(ops) {
return fmt.Errorf("shard %d: expected %d ops, got %d", shard, len(ops), len(ops2))
}
for i, op := range ops {
if err := op.Compare(ops2[i]); err != nil {
return fmt.Errorf("shard %d, op %d: %v", shard, i, err)
}
}
}
if len(other.Ops) != len(s.Ops) {
for shard := range other.Ops {
if _, ok := s.Ops[shard]; !ok {
return fmt.Errorf("shard %d missing in self", shard)
}
}
}
return nil
}

153
ingest/op_test.go
View file

@ -12,31 +12,29 @@
// See the License for the specific language governing permissions and
// limitations under the License.
package ingest_test
package ingest
import (
"math/rand"
"reflect"
"testing"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v2/shardwidth"
)
type opShardingTestCase struct {
name string
input ingest.Request
output *ingest.ShardedRequest
input *Request
output *ShardedRequest
}
var opShardingTestCases = []opShardingTestCase{
{
name: "sample",
input: ingest.Request{
Ops: []*ingest.Operation{
input: &Request{
Ops: []*Operation{
{
OpType: ingest.OpSet,
FieldOps: map[string]*ingest.FieldOperation{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{
"shard0": {
RecordIDs: []uint64{0, 1},
},
@ -55,8 +53,9 @@ var opShardingTestCases = []opShardingTestCase{
},
},
{
OpType: ingest.OpRemove,
FieldOps: map[string]*ingest.FieldOperation{
OpType: OpRemove,
Seq: 1,
FieldOps: map[string]*FieldOperation{
"shard0-2": {
RecordIDs: []uint64{1, 2<<shardwidth.Exponent + 1},
},
@ -64,12 +63,12 @@ var opShardingTestCases = []opShardingTestCase{
},
},
},
output: &ingest.ShardedRequest{
Ops: map[uint64][]*ingest.Operation{
output: &ShardedRequest{
Ops: map[uint64][]*Operation{
0: {
{
OpType: ingest.OpSet,
FieldOps: map[string]*ingest.FieldOperation{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{
"shard0": {
RecordIDs: []uint64{0, 1},
},
@ -81,8 +80,9 @@ var opShardingTestCases = []opShardingTestCase{
},
},
{
OpType: ingest.OpRemove,
FieldOps: map[string]*ingest.FieldOperation{
OpType: OpRemove,
Seq: 1,
FieldOps: map[string]*FieldOperation{
"shard0-2": {
RecordIDs: []uint64{1},
},
@ -91,8 +91,8 @@ var opShardingTestCases = []opShardingTestCase{
},
1: {
{
OpType: ingest.OpSet,
FieldOps: map[string]*ingest.FieldOperation{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{
"shard0-1": {
RecordIDs: []uint64{
1 << shardwidth.Exponent,
@ -109,8 +109,9 @@ var opShardingTestCases = []opShardingTestCase{
},
2: {
{
OpType: ingest.OpRemove,
FieldOps: map[string]*ingest.FieldOperation{
OpType: OpRemove,
Seq: 1,
FieldOps: map[string]*FieldOperation{
"shard0-2": {
RecordIDs: []uint64{2<<shardwidth.Exponent + 1},
},
@ -122,15 +123,115 @@ var opShardingTestCases = []opShardingTestCase{
},
}
func TestOpSharding(t *testing.T) {
func TestOpShardingSmall(t *testing.T) {
codec, _ := NewJSONCodec(nil)
_ = codec.AddIntField("shard0", nil)
_ = codec.AddIntField("shard1", nil)
_ = codec.AddIntField("shard0-1", nil)
_ = codec.AddIntField("shard0-2", nil)
fieldTypes := codec.FieldTypes()
for _, c := range opShardingTestCases {
sharded, err := c.input.ByShard()
sharded, err := c.input.ByShard(fieldTypes)
if err != nil {
t.Errorf("sharding: unexpected error %v", err)
}
if !reflect.DeepEqual(sharded, c.output) {
t.Fatalf("%s: expected %#v, got %#v", c.name, c.output, sharded)
if err := sharded.Compare(c.output); err != nil {
t.Fatalf("%s: shard: %v", c.name, err)
}
merged := sharded.merge()
if err := c.input.Compare(merged); err != nil {
t.Fatalf("%s: merge: %v", c.name, err)
}
}
}
func TestOpShardingLarge(t *testing.T) {
codec, err := NewJSONCodec(nil)
if err != nil {
t.Fatalf("creating codec: %v", err)
}
_ = codec.AddSetField("set", nil)
_ = codec.AddTimeQuantumField("tq", nil)
_ = codec.AddIntField("int", nil)
// and now we populate encodeTests[1] with a larger pool of data
// if this isn't large enough, the scattering across shards means
// we coincidentally end up with the time quantum field not getting
// tested on simpleSort.
const dataSize = 120000
shardCount := uint64(300) // shards we want to target
passes := uint64(0)
recordIDs := make([]uint64, dataSize)
values := make([]uint64, dataSize)
timeStamps := make([]int64, dataSize)
signedValues := make([]int64, dataSize)
for i := uint64(0); i < dataSize; i++ {
if (i % shardCount) == 0 {
passes++
}
recordIDs[i] = ((i % shardCount) << shardwidth.Exponent) + passes
values[i] = (i % 4)
timeStamps[i] = int64(1234567890e9 + (i * 100e9))
signedValues[i] = (int64(i) % 16) // no negative values because they won't work with keys
}
op := &Operation{
OpType: OpSet,
FieldOps: map[string]*FieldOperation{},
}
req := &Request{
Ops: []*Operation{op},
}
op.FieldOps["tq"] = &FieldOperation{
RecordIDs: append([]uint64{}, recordIDs...),
Values: append([]uint64{}, values...),
Signed: timeStamps,
}
op.FieldOps["int"] = &FieldOperation{
RecordIDs: recordIDs,
Signed: signedValues,
}
// for sets, we want to shuffle things into fewer shards, and ensure
// non-duplication of values within each record, but also have lots
// of duplication of record IDs in the low shards
recordIDs = make([]uint64, dataSize)
values = make([]uint64, dataSize)
valuesPerRecord := uint64(5)
recordsPerShard := dataSize / valuesPerRecord / 30
if recordsPerShard < 1 {
recordsPerShard = 1
}
shard := uint64(0)
nextID := uint64(0)
nextValue := uint64(0)
for i := uint64(0); i < dataSize; i++ {
recordIDs[i] = nextID
values[i] = nextValue + (i % valuesPerRecord)
nextValue++
if nextValue == valuesPerRecord {
nextValue = 0
nextID++
if nextID%(1<<shardwidth.Exponent) == recordsPerShard {
shard++
nextID = (shard << shardwidth.Exponent)
if valuesPerRecord > 1 {
valuesPerRecord--
}
}
}
}
op.FieldOps["set"] = &FieldOperation{
RecordIDs: recordIDs,
Values: values,
}
fieldTypes := codec.FieldTypes()
sharded, err := req.ByShard(fieldTypes)
if err != nil {
t.Errorf("sharding: unexpected error %v", err)
}
merged := sharded.merge()
if err := req.Compare(merged); err != nil {
t.Fatalf("merge comparison: %v", err)
}
}
@ -139,7 +240,7 @@ func TestFancySharding(t *testing.T) {
const recordCount = 5000
grr := rand.New(rand.NewSource(0))
for i := 0; i < 100; i++ {
f := &ingest.FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)}
f := &FieldOperation{RecordIDs: make([]uint64, recordCount), Values: make([]uint64, recordCount)}
shards := make([]int, shardLimit)
for j := range f.RecordIDs {
v := uint64(grr.Int63n(shardLimit << shardwidth.Exponent))

137
ingest/translate_test.go Normal file
View file

@ -0,0 +1,137 @@
// Copyright 2021 Molecula Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ingest
import (
"fmt"
"testing"
)
// stableTranslator implements a key translator that can be reused and
// will continue to give the same keys for the same values. Possibly
// surprisingly, it will invent new keys for IDs it is asked about but
// hasn't seen. This allows us to give a codec which would use keys on
// translation a request which contains arbitrary numbers, and request
// text that would parse into that request.
type stableTranslator struct {
in map[string]uint64
out map[uint64]string
next uint64
}
func (s *stableTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
ret := make(map[string]uint64, len(keys))
for _, key := range keys {
if existing, ok := s.in[key]; ok {
ret[key] = existing
continue
}
id := s.next
// but what if someone already translated that ID, so now it already
// exists?
if _, ok := s.out[id]; ok {
for k := range s.out {
if k > id {
id = k
}
}
// one larger than the largest we already have. this could
// wrap around, in which case, it's your own fault.
id++
}
s.next = id + 1
s.in[key] = id
s.out[id] = key
ret[key] = id
}
return ret, nil
}
func (s *stableTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
ret := make(map[uint64]string, len(ids))
for _, id := range ids {
if existing, ok := s.out[id]; ok {
ret[id] = existing
continue
}
key := fmt.Sprintf("k-%d", id)
s.in[key] = id
s.out[id] = key
ret[id] = key
if id >= s.next {
s.next = id + 1
}
}
return ret, nil
}
// newStableTranslator produces a translator which can translate forwards
// and backwards and invent new things if it needs to. Don't use this.
func newStableTranslator() *stableTranslator {
return &stableTranslator{
in: make(map[string]uint64),
out: make(map[uint64]string),
}
}
func TestTranslateReuse(t *testing.T) {
// the original stable-translator design had a flaw in that it
// assumed that each new ID would always come from a string translation,
// never from a key translation, and that they'd show up sequentially.
tr := newStableTranslator()
orig, err := tr.TranslateIDs(1, 2)
tr.next = 1 // intentionally break the translation logic for testing purposes
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
var s [6]string
stash := s[:0]
for _, v := range orig {
stash = append(stash, v)
}
for i := range s[len(orig):] {
stash = append(stash, fmt.Sprintf("key-%d", i))
}
keys, err := tr.TranslateKeys(stash...)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
ids := make([]uint64, 0, len(keys))
for _, v := range keys {
ids = append(ids, v)
}
idMap, err := tr.TranslateIDs(ids...)
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
for k, v := range keys {
if idMap[v] != k {
t.Fatalf("translate mismatch: keys %q->%d, ids %d->%q",
k, v, v, idMap[v])
}
}
for id, key := range idMap {
if keys[key] != id {
t.Fatalf("translate mismatch: ids %d->%q, keys %q->%d",
id, key, key, keys[key])
}
}
for id, key := range orig {
if keys[key] != id {
t.Fatalf("translate mismatch: original ids %d->%q, keys %q->%d",
id, key, key, keys[key])
}
}
}

57
ingest/vec.go
View file

@ -19,8 +19,6 @@ import (
"reflect"
"strconv"
"unsafe"
"github.com/pkg/errors"
)
// StringTable is a mapping of strings to temporary IDs.
@ -85,11 +83,8 @@ func (tbl *StringTable) IntID(in []byte) (int64, error) {
// integers and a translation function from strings to "real" keys, yields
// a translation/lookup slice. If it cannot translate all the keys, it
// returns an error.
//
// The lookup function corresponds to the FindKeys/CreateKeys methods of
// featurebase translators, by an AMAZING coincidence.
func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint64, error)) ([]uint64, error) {
lookedUp, err := lookup(tbl.names...)
func (tbl *StringTable) MakeIDMap(keys KeyTranslator) ([]uint64, error) {
lookedUp, err := keys.TranslateKeys(tbl.names...)
if err != nil {
return nil, err
}
@ -103,22 +98,38 @@ func MapForStringTable(tbl *StringTable, lookup func(...string) (map[string]uint
return out, nil
}
// TimeFormatForUnit returns the time transfer format (between the update encoder and the update applier) with appropriate resolution for a quantum unit.
func TimeFormatForUnit(unit rune) string {
switch unit {
case 'Y':
return "2006"
case 'M':
return "200601"
case 'D':
return "20060102"
case 'H':
return "2006010203"
default:
panic(errors.Errorf("invalid quantum unit: %q", unit))
// translateSigned replaces values from 0 to len(mapping)-1 with the
// elements of mapping. It yields an error if any values aren't
// mapped.
func translateSigned(mapping []uint64, values []int64) error {
oops := 0
for i, v := range values {
if v >= int64(len(mapping)) {
oops++
} else {
values[i] = int64(mapping[v])
}
}
if oops > 0 {
return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops)
}
return nil
}
// TODO: bool
// TODO: timestamp (just sugar on top of IntVector)
// translateUnsigned replaces values from 0 to len(mapping)-1 with the
// elements of mapping. It yields an error if any values aren't
// mapped.
func translateUnsigned(mapping []uint64, values []uint64) error {
oops := 0
for i, v := range values {
if v >= uint64(len(mapping)) {
oops++
} else {
values[i] = mapping[v]
}
}
if oops > 0 {
return fmt.Errorf("encountered %d out-of-range signed values when applying translation mapping", oops)
}
return nil
}

114
ingest/vec_test.go Normal file
View file

@ -0,0 +1,114 @@
// Copyright 2021 Molecula Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package ingest
import (
"errors"
"testing"
)
type badTranslator struct{}
func (b badTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
if len(keys) == 0 {
return nil, errors.New("no keys")
}
m := make(map[string]uint64)
skip := true
for i, k := range keys {
if skip {
skip = false
continue
}
m[k] = uint64(i)
}
out := make([]uint64, len(keys)-1)
for i := range out {
out[i] = uint64(i)
}
return nil, nil
}
func (b badTranslator) TranslateIDs(...uint64) (map[uint64]string, error) {
return nil, nil
}
func TestStringTableErrors(t *testing.T) {
tbl := NewStringTable()
btr := badTranslator{}
_, keyErr := tbl.MakeIDMap(btr)
if keyErr == nil {
t.Fatalf("expected error passed up from failed translate, didn't get it")
}
a1, err := tbl.ID([]byte("a"))
if err != nil {
t.Fatalf("getting translation for key: %v", err)
}
b1, err := tbl.ID([]byte("b"))
if err != nil {
t.Fatalf("getting translation for key: %v", err)
}
_, keyErr = tbl.MakeIDMap(btr)
if keyErr == nil {
t.Fatalf("expected error for short translate, didn't get it")
}
tr := newStableTranslator()
_, err = tr.TranslateKeys("c", "d")
if err != nil {
t.Fatalf("translating stray keys: %v", err)
}
m, err := tbl.MakeIDMap(tr)
if err != nil {
t.Fatalf("creating lookup: %v", err)
}
var y = []uint64{a1, b1}
err = translateUnsigned(m, y)
if err != nil {
t.Fatalf("unexpected unsigned translation error: %v", err)
}
trResults, err := tr.TranslateKeys("a", "b")
if err != nil {
t.Fatalf("unexpected translation error: %v", err)
}
if y[0] != trResults["a"] {
t.Fatalf("expected %d, got %d", trResults["a"], y[0])
}
if y[1] != trResults["b"] {
t.Fatalf("expected %d, got %d", trResults["b"], y[1])
}
y[0] = a1
y[1] = (a1 + b1 + 1) // assumed not to be any of them
err = translateUnsigned(m, y)
if err == nil {
t.Fatalf("no error from translating invalid table")
}
z := []int64{int64(a1), int64(b1)}
err = translateSigned(m, z)
if err != nil {
t.Fatalf("unexpected unsigned translation error: %v", err)
}
if uint64(z[0]) != trResults["a"] {
t.Fatalf("expected %d, got %d", trResults["a"], z[0])
}
if uint64(z[1]) != trResults["b"] {
t.Fatalf("expected %d, got %d", trResults["b"], z[1])
}
z[0] = int64(a1)
z[1] = int64(a1 + b1 + 1) // assumed not to be any of them
err = translateSigned(m, z)
if err == nil {
t.Fatalf("no error from translating invalid table")
}
}

11
ingest_test.go
View file

@ -26,9 +26,7 @@ import (
"testing"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v2/server"
"github.com/molecula/featurebase/v2/test"
"github.com/pkg/errors"
)
@ -448,14 +446,7 @@ func TestIngestTestcases(t *testing.T) {
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
c := test.MustRunCluster(t, 1,
[]server.CommandOption{
server.OptCommandServerOptions(
pilosa.OptServerNodeID("node0"),
pilosa.OptServerClusterHasher(&offsetModHasher{}),
pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)),
)},
)
c := test.MustRunCluster(t, 3)
defer c.Close()
coord := c.GetPrimary()

4
internal/clustertests/cluster_test.go
View file

@ -21,7 +21,7 @@ import (
"testing"
"time"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/disco"
picli "github.com/molecula/featurebase/v2/http"
)
@ -115,7 +115,7 @@ func waitForStatus(t *testing.T, stator func(context.Context) (string, error), s
if err != nil {
t.Logf("Status (try %d/%d): %v (retrying in %s)", i, n, err, sleep.String())
} else {
t.Logf("Status (try %d/%d): %s (retrying in %s)", i, n, s, sleep.String())
t.Logf("Status (try %d/%d): curr: %s, expect: %s (retrying in %s)", i, n, s, status, sleep.String())
}
if s == status {
return

7
internal/clustertests/docker-compose.yml
View file

@ -14,6 +14,7 @@ services:
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
- PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa1:10301
- PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
- PILOSA_CLUSTER_REPLICAS=3
networks:
- pilosanet
command:
@ -32,6 +33,7 @@ services:
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
- PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa2:10301
- PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
- PILOSA_CLUSTER_REPLICAS=3
networks:
- pilosanet
command:
@ -50,6 +52,7 @@ services:
- PILOSA_ETCD_LISTEN_PEER_ADDRESS=http://0.0.0.0:10301
- PILOSA_ETCD_ADVERTISE_PEER_ADDRESS=http://pilosa3:10301
- PILOSA_ETCD_INITIAL_CLUSTER=pilosa1=http://pilosa1:10301,pilosa2=http://pilosa2:10301,pilosa3=http://pilosa3:10301
- PILOSA_CLUSTER_REPLICAS=3
networks:
- pilosanet
command:
@ -57,6 +60,10 @@ services:
client1:
build:
context: .
depends_on:
- "pilosa1"
- "pilosa2"
- "pilosa3"
environment:
- ENABLE_PILOSA_CLUSTER_TESTS=1
- GO111MODULE=on

412
internal/clustertests/pause_node_test.go Normal file
View file

@ -0,0 +1,412 @@
// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package clustertest
import (
"context"
"fmt"
"hash/fnv"
"io/ioutil"
"math/rand"
"os"
"os/exec"
"path/filepath"
"strconv"
"testing"
"time"
pilosa "github.com/molecula/featurebase/v2"
boltdb "github.com/molecula/featurebase/v2/boltdb"
"github.com/molecula/featurebase/v2/disco"
"github.com/molecula/featurebase/v2/http"
picli "github.com/molecula/featurebase/v2/http"
"github.com/molecula/featurebase/v2/net"
"github.com/molecula/featurebase/v2/topology"
"github.com/pkg/errors"
)
func sendCmd(cmd string, args ...string) error {
pcmd := exec.Command(cmd, args...)
pcmd.Stdout = os.Stdout
pcmd.Stderr = os.Stderr
err := pcmd.Start()
if err != nil {
return errors.Wrap(err, "starting cmd")
}
err = pcmd.Wait()
if err != nil {
return errors.Wrap(err, "waiting on cmd")
}
return nil
}
func unpauseNode(node string) error {
unpauseArgs := []string{"container", "unpause", "clustertests_" + node + "_1"}
return sendCmd("docker", unpauseArgs...)
}
func pauseNode(node string) error {
pauseArgs := []string{"container", "pause", "clustertests_" + node + "_1"}
return sendCmd("docker", pauseArgs...)
}
type keyInserter struct {
client *http.InternalClient
uri *net.URI
index string
keys []string
}
func (ki keyInserter) insertKeys(ctx context.Context) (map[string]uint64, error) {
ts, err := ki.client.CreateIndexKeysNode(ctx, ki.uri, ki.index, ki.keys...)
return ts, err
}
func getAddress(node string) string {
return node + ":10101"
}
func getClients(addrs []string) ([]*http.InternalClient, error) {
clients := make([]*http.InternalClient, 0, len(addrs))
for _, addr := range addrs {
c, err := picli.NewInternalClient(addr, picli.GetHTTPClient(nil))
if err != nil {
return nil, err
}
clients = append(clients, c)
}
return clients, nil
}
func getURIsFromAddresses(addrs []string) ([]*net.URI, error) {
uris := make([]*net.URI, 0, len(addrs))
for _, addr := range addrs {
uri, err := net.NewURIFromAddress(addr)
if err != nil {
return nil, err
}
uris = append(uris, uri)
}
return uris, nil
}
func readIndexTranslateData(ctx context.Context, client *picli.InternalClient, dirPath, index string, partition int) error {
// read translateStore contents from endpoint
r, err := client.IndexTranslateDataReader(ctx, index, partition)
if err != nil {
return err
}
buf, err := ioutil.ReadAll(r)
if err != nil {
return err
}
r.Close()
// create file and write contents to it
filename := strconv.FormatInt(int64(partition), 10)
filePath := filepath.Join(dirPath, filename)
file, err := os.Create(filePath)
if err != nil {
return err
}
_, err = file.Write(buf)
if err != nil {
file.Close()
return err
}
err = file.Sync()
if err != nil {
file.Close()
return err
}
file.Close()
return nil
}
func openTranslateStores(dirPath, index string) (map[int]pilosa.TranslateStore, error) {
dirEntries, err := ioutil.ReadDir(dirPath)
if err != nil {
return nil, err
}
// in case of error, close any translateStore that has been opened
rollback := make([]pilosa.TranslateStore, 0, topology.DefaultPartitionN)
defer func() {
for _, ts := range rollback {
_ = ts.Close()
}
}()
// filter out non-file entries
filePaths := make([]string, 0, len(dirEntries))
for _, entry := range dirEntries {
if entry.Mode().IsDir() {
continue
}
filePath := filepath.Join(dirPath, entry.Name())
filePaths = append(filePaths, filePath)
}
translateStores := make(map[int]pilosa.TranslateStore)
for _, filePath := range filePaths {
// extract partition number from filename
file := filepath.Base(filePath)
partition, err := strconv.Atoi(file)
if err != nil {
return nil, err
}
// open bolt db
ts, err := boltdb.OpenTranslateStore(filePath, index, "", partition, topology.DefaultPartitionN, false)
ts.SetReadOnly(true)
if err != nil {
return nil, err
}
translateStores[partition] = ts
rollback = append(rollback, ts)
}
rollback = nil
return translateStores, nil
}
var errOpRetriable = errors.New("If operation failed on this error, it can be retried")
func verifyNodeHasGivenKeys(ctx context.Context, node, index, dirPath string, keys []string) error {
// get client that's connected to node
address := getAddress(node)
client, err := picli.NewInternalClient(address, picli.GetHTTPClient(nil))
if err != nil {
return err
}
// create dir to store boltdbs for this node
nodeDirPath := filepath.Join(dirPath, node)
err = os.Mkdir(nodeDirPath, 0755)
if err != nil {
return err
}
// read in all the translate stores for each partition
for partition := 0; partition < topology.DefaultPartitionN; partition++ {
err := readIndexTranslateData(ctx, client, nodeDirPath, index, partition)
if err != nil {
return err
}
}
// open all the translate stores
translateStores, err := openTranslateStores(nodeDirPath, index)
if err != nil {
return err
}
// close all the translate stores on complete
defer func() {
for _, ts := range translateStores {
ts.Close()
}
}()
// merge all translations
merged := make(map[string]uint64)
for _, ts := range translateStores {
entries, err := ts.FindKeys(keys...)
if err != nil {
return err
}
for key, id := range entries {
merged[key] = id
}
}
// check that all expected keys present in node
if len(merged) != len(keys) {
msg := fmt.Sprintf("entries in node %s: %d. keys inserted: %d",
node, len(merged), len(keys))
return errors.Wrap(errOpRetriable, msg)
}
for _, k := range keys {
if _, ok := merged[k]; !ok {
msg := fmt.Sprintf("Key '%s' not present in node %s", k, node)
return errors.Wrap(errOpRetriable, msg)
}
}
return nil
}
func genKeys(count, maxTries int, keyToNode func(string) string, filterOut []string) []string {
exclusionSet := make(map[string]struct{})
for _, n := range filterOut {
exclusionSet[n] = struct{}{}
}
keys := make([]string, 0, count)
i := 0
for {
if i >= maxTries {
break
}
key := fmt.Sprintf("key-%d", i)
i++
// get primary node for this key
node := keyToNode(key)
// check if we should exclude this key
if _, ok := exclusionSet[node]; ok {
continue
}
// add key
keys = append(keys, key)
if len(keys) >= count {
return keys
}
}
return keys
}
func TestPauseReplica(t *testing.T) {
if os.Getenv("ENABLE_PILOSA_CLUSTER_TESTS") != "1" {
t.Skip("pilosa cluster tests for replication when a replica is paused are not enabled")
}
// configurations for test
nodeNames := []string{"pilosa1", "pilosa2", "pilosa3"}
nodeToPause := "pilosa3"
addresses := make([]string, len(nodeNames))
for i, node := range nodeNames {
addresses[i] = getAddress(node)
}
clients, err := getClients(addresses)
if err != nil {
t.Fatalf("on init clients from addresses: %v, %v", addresses, err)
}
cli := clients[0]
uris, err := getURIsFromAddresses(addresses)
if err != nil {
t.Fatalf("on init clients from addresses: %v, %v", addresses, err)
}
uri := uris[0]
ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
t.Log("start Client")
// first achieve normal cluster status
waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second)
// create keyed index
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
index := fmt.Sprintf("keyed-index-%d", rng.Int63())
err = cli.EnsureIndex(ctx, index, pilosa.IndexOptions{
Keys: true,
})
if err != nil {
t.Fatalf("creating/asserting index: %v", err)
}
// generate mapping from partition to primary node
partitionToNode := make([]string, topology.DefaultPartitionN)
for partition := 0; partition < topology.DefaultPartitionN; partition++ {
nodes, err := cli.PartitionNodes(ctx, partition)
if err != nil {
t.Fatal(err)
}
partitionToNode[partition] = nodes[0].URI.Host
}
// generate random keys to insert
// no guarantee that all keys expected to be generated don't fall
// in nodes to be filtered out
keyCount := 100
maxKeyGenTries := 1000
filterOutKeysFromTheseNodes := []string{nodeToPause}
keyToNode := func(key string) string {
// get partition for this key
h := fnv.New64a()
_, _ = h.Write([]byte(index))
_, _ = h.Write([]byte(key))
partition := int(h.Sum64() % uint64(topology.DefaultPartitionN))
// get node for this partition
return partitionToNode[partition]
}
keys := genKeys(keyCount, maxKeyGenTries, keyToNode, filterOutKeysFromTheseNodes)
// pause node
t.Logf("pause %s", nodeToPause)
err = pauseNode(nodeToPause)
if err != nil {
t.Fatalf("error on pause node %s: %v", nodeToPause, err)
}
// insert keys
t.Log("start insert")
ts, err := keyInserter{
client: cli,
uri: uri,
index: index,
keys: keys,
}.insertKeys(ctx)
if err != nil {
t.Fatalf("Error: inserting index keys for index(%s) send to node(%s): %v", index, uri.String(), err)
}
t.Logf("successfully end insert: %v", len(ts))
// wait for cluster status to be non-normal
waitForStatus(t, cli.Status, string(disco.ClusterStateDegraded), 30, 1*time.Second)
// wait for cluster status to get back to normal
t.Logf("unpause %s", nodeToPause)
err = unpauseNode(nodeToPause)
if err != nil {
t.Fatalf("error on unpause node %s: %v", nodeToPause, err)
}
waitForStatus(t, cli.Status, string(disco.ClusterStateNormal), 30, 1*time.Second)
// set up directory to store keys
basePath := "."
keysDirName := "index_keys"
dirPath, err := filepath.Abs(basePath)
if err != nil {
t.Fatal(err)
}
dirPath = filepath.Join(dirPath, keysDirName)
err = os.Mkdir(dirPath, 0755)
if err != nil {
t.Fatal(err)
}
maxRetries := 10
durationInBetweenRetries := 5 * time.Second
for _, node := range nodeNames {
try := 1
retries:
for {
err = verifyNodeHasGivenKeys(ctx, node, index, dirPath, keys)
if err == nil {
break retries
}
if try <= maxRetries && errors.Is(err, errOpRetriable) {
try++
t.Logf("node %s, retry verify key replication: (%d/%d) after %v\n",
node, try, maxRetries, durationInBetweenRetries)
time.Sleep(durationInBetweenRetries)
continue
}
t.Fatal(errors.Wrap(err, fmt.Sprintf("try: (%d/%d)", try, maxRetries)))
}
}
cancel()
t.Log("Done")
}

1852
pb/private.pb.go
View file

File diff suppressed because it is too large Load diff

23
pb/private.proto
View file

@ -234,4 +234,25 @@ message ResizeAbortMessage {
message ResizeNodeMessage {
string NodeID = 1;
string Action = 2;
}
}
message FieldOperation {
repeated uint64 RecordIDs = 1;
repeated uint64 Values = 2;
repeated int64 Signed = 3;
}
message ShardIngestOperation {
string OpType = 1;
repeated uint64 ClearRecordIDs = 2;
repeated string ClearFields = 3;
map <string, FieldOperation> FieldOps = 4;
}
message ShardIngestOperations {
repeated ShardIngestOperation Ops = 1;
}
message ShardedIngestRequest {
map <uint64, ShardIngestOperations> Ops = 1;
}

170
pb/public.pb.go
View file

@ -5980,10 +5980,7 @@ func (m *Row) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -6068,10 +6065,7 @@ func (m *RowMatrix) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -6194,10 +6188,7 @@ func (m *SignedRow) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -6356,10 +6347,7 @@ func (m *RowIdentifiers) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -6486,10 +6474,7 @@ func (m *IDList) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -6593,10 +6578,7 @@ func (m *ExtractedIDColumn) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -6713,10 +6695,7 @@ func (m *ExtractedIDMatrix) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -6799,10 +6778,7 @@ func (m *KeyList) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7016,10 +6992,7 @@ func (m *ExtractedTableValue) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7156,10 +7129,7 @@ func (m *ExtractedTableColumn) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7274,10 +7244,7 @@ func (m *ExtractedTableField) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7396,10 +7363,7 @@ func (m *ExtractedTable) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7520,10 +7484,7 @@ func (m *Pair) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7642,10 +7603,7 @@ func (m *PairField) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7762,10 +7720,7 @@ func (m *PairsField) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -7835,10 +7790,7 @@ func (m *Int64) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8008,10 +7960,7 @@ func (m *FieldRow) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8134,10 +8083,7 @@ func (m *GroupCount) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8273,10 +8219,7 @@ func (m *ValCount) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8365,10 +8308,7 @@ func (m *Decimal) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8620,10 +8560,7 @@ func (m *QueryRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -8740,10 +8677,7 @@ func (m *QueryResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -9356,10 +9290,7 @@ func (m *QueryResult) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -9843,10 +9774,7 @@ func (m *ImportRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -10308,10 +10236,7 @@ func (m *ImportValueRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -10481,10 +10406,7 @@ func (m *AtomicRecord) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -10567,10 +10489,7 @@ func (m *AtomicImportResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -10737,10 +10656,7 @@ func (m *TranslateKeysRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -10867,10 +10783,7 @@ func (m *TranslateKeysResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11061,10 +10974,7 @@ func (m *TranslateIDsRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11147,10 +11057,7 @@ func (m *TranslateIDsResponse) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11267,10 +11174,7 @@ func (m *ImportRoaringRequestView) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11484,10 +11388,7 @@ func (m *ImportRoaringRequest) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {
@ -11604,10 +11505,7 @@ func (m *GroupCounts) Unmarshal(dAtA []byte) error {
if err != nil {
return err
}
if skippy < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) < 0 {
if (skippy < 0) || (iNdEx+skippy) < 0 {
return ErrInvalidLengthPublic
}
if (iNdEx + skippy) > l {

652
pg/lookerToFeaturebase.md Normal file
View file

@ -0,0 +1,652 @@
```mermaid
sequenceDiagram
participant 213070643358480 as c0
participant 213070643358482 as c1
participant 213070643358484 as c2
participant 213070643358486 as c3
participant 213070643358488 as c4
participant 213070643358490 as c5
participant 213070643358492 as c6
participant 213070643358494 as c7
participant 213070643358496 as c8
participant 213070643358498 as c9
participant 213070643358500 as c10
participant 213070643358502 as c11
213070643358480->>server:+SSL REQUEST
server-->>213070643358480:-SSL BACKEND ANSWER: N
213070643358480->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358480:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358480:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358480:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358480:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358480:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358480:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358480:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358480:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358480:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358480:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358480:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358480:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358480:-BACKEND KEY DATA pid=1459324827, key=1506254533
server-->>213070643358480:-READY FOR QUERY type=<IDLE>
213070643358480->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358480->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358480->>server:+EXECUTE name='', nb_rows=1
213070643358480->>server:+SYNC
server-->>213070643358480:-PARSE COMPLETE
server-->>213070643358480:-BIND COMPLETE
server-->>213070643358480:-COMMAND COMPLETE command='SET'
server-->>213070643358480:-READY FOR QUERY type=<IDLE>
213070643358480->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358480->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358480->>server:+EXECUTE name='', nb_rows=1
213070643358480->>server:+SYNC
server-->>213070643358480:-PARSE COMPLETE
server-->>213070643358480:-BIND COMPLETE
server-->>213070643358480:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358480:-COMMAND COMPLETE command='SET'
server-->>213070643358480:-READY FOR QUERY type=<IDLE>
213070643358480->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358480->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358480->>server:+DESCRIBE kind='P', name=''
213070643358480->>server:+EXECUTE name='', nb_rows=1
213070643358480->>server:+SYNC
server-->>213070643358480:-PARSE COMPLETE
server-->>213070643358480:-BIND COMPLETE
server-->>213070643358480:-NO DATA
server-->>213070643358480:-EMPTY QUERY RESPONSE
server-->>213070643358480:-READY FOR QUERY type=<IDLE>
213070643358480->>server:+DISCONNECT
213070643358482->>server:+SSL REQUEST
server-->>213070643358482:-SSL BACKEND ANSWER: N
213070643358482->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358482:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358482:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358482:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358482:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358482:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358482:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358482:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358482:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358482:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358482:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358482:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358482:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358482:-BACKEND KEY DATA pid=1742342691, key=1299317425
server-->>213070643358482:-READY FOR QUERY type=<IDLE>
213070643358482->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358482->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358482->>server:+EXECUTE name='', nb_rows=1
213070643358482->>server:+SYNC
server-->>213070643358482:-PARSE COMPLETE
server-->>213070643358482:-BIND COMPLETE
server-->>213070643358482:-COMMAND COMPLETE command='SET'
server-->>213070643358482:-READY FOR QUERY type=<IDLE>
213070643358482->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358482->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358482->>server:+EXECUTE name='', nb_rows=1
213070643358482->>server:+SYNC
server-->>213070643358482:-PARSE COMPLETE
server-->>213070643358482:-BIND COMPLETE
server-->>213070643358482:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358482:-COMMAND COMPLETE command='SET'
server-->>213070643358482:-READY FOR QUERY type=<IDLE>
213070643358482->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358482->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358482->>server:+DESCRIBE kind='P', name=''
213070643358482->>server:+EXECUTE name='', nb_rows=1
213070643358482->>server:+SYNC
server-->>213070643358482:-PARSE COMPLETE
server-->>213070643358482:-BIND COMPLETE
server-->>213070643358482:-NO DATA
server-->>213070643358482:-EMPTY QUERY RESPONSE
server-->>213070643358482:-READY FOR QUERY type=<IDLE>
213070643358482->>server:+DISCONNECT
213070643358484->>server:+SSL REQUEST
server-->>213070643358484:-SSL BACKEND ANSWER: N
213070643358484->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358484:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358484:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358484:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358484:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358484:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358484:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358484:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358484:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358484:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358484:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358484:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358484:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358484:-BACKEND KEY DATA pid=700852804, key=1377869267
server-->>213070643358484:-READY FOR QUERY type=<IDLE>
213070643358484->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358484->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358484->>server:+EXECUTE name='', nb_rows=1
213070643358484->>server:+SYNC
server-->>213070643358484:-PARSE COMPLETE
server-->>213070643358484:-BIND COMPLETE
server-->>213070643358484:-COMMAND COMPLETE command='SET'
server-->>213070643358484:-READY FOR QUERY type=<IDLE>
213070643358484->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358484->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358484->>server:+EXECUTE name='', nb_rows=1
213070643358484->>server:+SYNC
server-->>213070643358484:-PARSE COMPLETE
server-->>213070643358484:-BIND COMPLETE
server-->>213070643358484:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358484:-COMMAND COMPLETE command='SET'
server-->>213070643358484:-READY FOR QUERY type=<IDLE>
213070643358484->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358484->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358484->>server:+DESCRIBE kind='P', name=''
213070643358484->>server:+EXECUTE name='', nb_rows=1
213070643358484->>server:+SYNC
server-->>213070643358484:-PARSE COMPLETE
server-->>213070643358484:-BIND COMPLETE
server-->>213070643358484:-NO DATA
server-->>213070643358484:-EMPTY QUERY RESPONSE
server-->>213070643358484:-READY FOR QUERY type=<IDLE>
213070643358484->>server:+DISCONNECT
213070643358486->>server:+SSL REQUEST
server-->>213070643358486:-SSL BACKEND ANSWER: N
213070643358486->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358486:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358486:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358486:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358486:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358486:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358486:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358486:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358486:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358486:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358486:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358486:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358486:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358486:-BACKEND KEY DATA pid=413235241, key=1302652759
server-->>213070643358486:-READY FOR QUERY type=<IDLE>
213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358486->>server:+EXECUTE name='', nb_rows=1
213070643358486->>server:+SYNC
server-->>213070643358486:-PARSE COMPLETE
server-->>213070643358486:-BIND COMPLETE
server-->>213070643358486:-COMMAND COMPLETE command='SET'
server-->>213070643358486:-READY FOR QUERY type=<IDLE>
213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358486->>server:+EXECUTE name='', nb_rows=1
213070643358486->>server:+SYNC
server-->>213070643358486:-PARSE COMPLETE
server-->>213070643358486:-BIND COMPLETE
server-->>213070643358486:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358486:-COMMAND COMPLETE command='SET'
server-->>213070643358486:-READY FOR QUERY type=<IDLE>
213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358486->>server:+DESCRIBE kind='P', name=''
213070643358486->>server:+EXECUTE name='', nb_rows=1
213070643358486->>server:+SYNC
server-->>213070643358486:-PARSE COMPLETE
server-->>213070643358486:-BIND COMPLETE
server-->>213070643358486:-NO DATA
server-->>213070643358486:-EMPTY QUERY RESPONSE
server-->>213070643358486:-READY FOR QUERY type=<IDLE>
213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SELECT pg_backend_pid()
213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358486->>server:+DESCRIBE kind='P', name=''
213070643358486->>server:+EXECUTE name='', nb_rows=0
213070643358486->>server:+SYNC
server-->>213070643358486:-PARSE COMPLETE
server-->>213070643358486:-BIND COMPLETE
server-->>213070643358486:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='pg_backend_pid' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643358486:-DATA ROW num_values=1 ---[Value 0001]--- length=9 value='413235241'
server-->>213070643358486:-COMMAND COMPLETE command='SELECT'
server-->>213070643358486:-READY FOR QUERY type=<IDLE>
213070643358486->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358486->>server:+DESCRIBE kind='P', name=''
213070643358486->>server:+EXECUTE name='', nb_rows=0
213070643358486->>server:+SYNC
server-->>213070643358486:-PARSE COMPLETE
server-->>213070643358486:-BIND COMPLETE
server-->>213070643358486:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643358486:-DATA ROW num_values=1 ---[Value 0001]--- length=27 value='PostgresSQL 13.0 (molecula)'
server-->>213070643358486:-COMMAND COMPLETE command='SELECT'
server-->>213070643358486:-READY FOR QUERY type=<IDLE>
213070643358486->>server:+PARSE name='', num_params=0, params_type=, query= SELECT COUNT(*) FROM pg_type AS t0, pg_aggregate AS t1, pg_settings AS t2, pg_settings AS t3
213070643358486->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358486->>server:+DESCRIBE kind='P', name=''
213070643358486->>server:+EXECUTE name='', nb_rows=0
213070643358486->>server:+SYNC
213070643358488->>server:+SSL REQUEST
server-->>213070643358488:-SSL BACKEND ANSWER: N
213070643358488->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358488:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358488:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358488:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358488:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358488:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358488:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358488:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358488:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358488:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358488:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358488:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358488:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358488:-BACKEND KEY DATA pid=180554708, key=1504602717
server-->>213070643358488:-READY FOR QUERY type=<IDLE>
213070643358488->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358488->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358488->>server:+EXECUTE name='', nb_rows=1
213070643358488->>server:+SYNC
server-->>213070643358488:-PARSE COMPLETE
server-->>213070643358488:-BIND COMPLETE
server-->>213070643358488:-COMMAND COMPLETE command='SET'
server-->>213070643358488:-READY FOR QUERY type=<IDLE>
213070643358488->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358488->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358488->>server:+EXECUTE name='', nb_rows=1
213070643358488->>server:+SYNC
server-->>213070643358488:-PARSE COMPLETE
server-->>213070643358488:-BIND COMPLETE
server-->>213070643358488:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358488:-COMMAND COMPLETE command='SET'
server-->>213070643358488:-READY FOR QUERY type=<IDLE>
213070643358488->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358488->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358488->>server:+DESCRIBE kind='P', name=''
213070643358488->>server:+EXECUTE name='', nb_rows=1
213070643358488->>server:+SYNC
server-->>213070643358488:-PARSE COMPLETE
server-->>213070643358488:-BIND COMPLETE
server-->>213070643358488:-NO DATA
server-->>213070643358488:-EMPTY QUERY RESPONSE
server-->>213070643358488:-READY FOR QUERY type=<IDLE>
213070643358488->>server:+DISCONNECT
213070643358490->>server:+SSL REQUEST
server-->>213070643358490:-SSL BACKEND ANSWER: N
213070643358490->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358490:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358490:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358490:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358490:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358490:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358490:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358490:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358490:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358490:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358490:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358490:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358490:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358490:-BACKEND KEY DATA pid=1713101217, key=29850369
server-->>213070643358490:-READY FOR QUERY type=<IDLE>
213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358490->>server:+EXECUTE name='', nb_rows=1
213070643358490->>server:+SYNC
server-->>213070643358490:-PARSE COMPLETE
server-->>213070643358490:-BIND COMPLETE
server-->>213070643358490:-COMMAND COMPLETE command='SET'
server-->>213070643358490:-READY FOR QUERY type=<IDLE>
213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358490->>server:+EXECUTE name='', nb_rows=1
213070643358490->>server:+SYNC
server-->>213070643358490:-PARSE COMPLETE
server-->>213070643358490:-BIND COMPLETE
server-->>213070643358490:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358490:-COMMAND COMPLETE command='SET'
server-->>213070643358490:-READY FOR QUERY type=<IDLE>
213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358490->>server:+DESCRIBE kind='P', name=''
213070643358490->>server:+EXECUTE name='', nb_rows=1
213070643358490->>server:+SYNC
server-->>213070643358490:-PARSE COMPLETE
server-->>213070643358490:-BIND COMPLETE
server-->>213070643358490:-NO DATA
server-->>213070643358490:-EMPTY QUERY RESPONSE
server-->>213070643358490:-READY FOR QUERY type=<IDLE>
213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358490->>server:+DESCRIBE kind='P', name=''
213070643358490->>server:+EXECUTE name='', nb_rows=0
213070643358490->>server:+SYNC
server-->>213070643358490:-PARSE COMPLETE
server-->>213070643358490:-BIND COMPLETE
server-->>213070643358490:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643358490:-DATA ROW num_values=1 ---[Value 0001]--- length=27 value='PostgresSQL 13.0 (molecula)'
server-->>213070643358490:-COMMAND COMPLETE command='SELECT'
server-->>213070643358490:-READY FOR QUERY type=<IDLE>
213070643358490->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358490->>server:+DESCRIBE kind='P', name=''
213070643358490->>server:+EXECUTE name='', nb_rows=1
213070643358490->>server:+SYNC
server-->>213070643358490:-PARSE COMPLETE
server-->>213070643358490:-BIND COMPLETE
server-->>213070643358490:-NO DATA
server-->>213070643358490:-EMPTY QUERY RESPONSE
server-->>213070643358490:-READY FOR QUERY type=<IDLE>
213070643358490->>server:+PARSE name='', num_params=0, params_type=, query= SELECT pid as id, query as stmt, EXTRACT(seconds from query_start - NOW()) as elapsed_time FROM pg_stat_activity WHERE usename='docker'
213070643358490->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358490->>server:+DESCRIBE kind='P', name=''
213070643358490->>server:+EXECUTE name='', nb_rows=0
213070643358490->>server:+SYNC
server-->>213070643358490:-PARSE COMPLETE
server-->>213070643358490:-BIND COMPLETE
server-->>213070643358490:-ROW DESCRIPTION: num_fields=3 ---[Field 01]--- name='id' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0 ---[Field 02]--- name='stmt' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0 ---[Field 03]--- name='elapsed_time' type=701 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643358490:-DATA ROW num_values=3 ---[Value 0001]--- length=9 value='413235241' ---[Value 0002]--- length=148 value=' SELECT COUNT(*). FROM pg_type AS t0,. pg_aggregate AS t1,. pg_settings AS t2,. pg_settings AS t3.' ---[Value 0003]--- length=11 value='1.311371547'
server-->>213070643358490:-DATA ROW num_values=3 ---[Value 0001]--- length=10 value='1713101217' ---[Value 0002]--- length=190 value=' SELECT pid as id,. query as stmt,. EXTRACT(seconds from query_start - NOW()) as elapsed_time. FROM pg_stat_activity. WHERE usename='docker'.' ---[Value 0003]--- length=11 value='0.000169969'
server-->>213070643358490:-COMMAND COMPLETE command='SELECT'
server-->>213070643358490:-READY FOR QUERY type=<IDLE>
213070643358492->>server:+SSL REQUEST
server-->>213070643358492:-SSL BACKEND ANSWER: N
213070643358492->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358492:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358492:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358492:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358492:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358492:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358492:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358492:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358492:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358492:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358492:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358492:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358492:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358492:-BACKEND KEY DATA pid=2034361563, key=1419995666
server-->>213070643358492:-READY FOR QUERY type=<IDLE>
213070643358492->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358492->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358492->>server:+EXECUTE name='', nb_rows=1
213070643358492->>server:+SYNC
server-->>213070643358492:-PARSE COMPLETE
server-->>213070643358492:-BIND COMPLETE
server-->>213070643358492:-COMMAND COMPLETE command='SET'
server-->>213070643358492:-READY FOR QUERY type=<IDLE>
213070643358492->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358492->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358492->>server:+EXECUTE name='', nb_rows=1
213070643358492->>server:+SYNC
server-->>213070643358492:-PARSE COMPLETE
server-->>213070643358492:-BIND COMPLETE
server-->>213070643358492:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358492:-COMMAND COMPLETE command='SET'
server-->>213070643358492:-READY FOR QUERY type=<IDLE>
213070643358492->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358492->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358492->>server:+DESCRIBE kind='P', name=''
213070643358492->>server:+EXECUTE name='', nb_rows=1
213070643358492->>server:+SYNC
server-->>213070643358492:-PARSE COMPLETE
server-->>213070643358492:-BIND COMPLETE
server-->>213070643358492:-NO DATA
server-->>213070643358492:-EMPTY QUERY RESPONSE
server-->>213070643358492:-READY FOR QUERY type=<IDLE>
213070643358492->>server:+DISCONNECT
213070643358494->>server:+SSL REQUEST
server-->>213070643358494:-SSL BACKEND ANSWER: N
213070643358494->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358494:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358494:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358494:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358494:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358494:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358494:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358494:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358494:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358494:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358494:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358494:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358494:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358494:-BACKEND KEY DATA pid=964429462, key=1673405115
server-->>213070643358494:-READY FOR QUERY type=<IDLE>
213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358494->>server:+EXECUTE name='', nb_rows=1
213070643358494->>server:+SYNC
server-->>213070643358494:-PARSE COMPLETE
server-->>213070643358494:-BIND COMPLETE
server-->>213070643358494:-COMMAND COMPLETE command='SET'
server-->>213070643358494:-READY FOR QUERY type=<IDLE>
213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358494->>server:+EXECUTE name='', nb_rows=1
213070643358494->>server:+SYNC
server-->>213070643358494:-PARSE COMPLETE
server-->>213070643358494:-BIND COMPLETE
server-->>213070643358494:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358494:-COMMAND COMPLETE command='SET'
server-->>213070643358494:-READY FOR QUERY type=<IDLE>
213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358494->>server:+DESCRIBE kind='P', name=''
213070643358494->>server:+EXECUTE name='', nb_rows=1
213070643358494->>server:+SYNC
server-->>213070643358494:-PARSE COMPLETE
server-->>213070643358494:-BIND COMPLETE
server-->>213070643358494:-NO DATA
server-->>213070643358494:-EMPTY QUERY RESPONSE
server-->>213070643358494:-READY FOR QUERY type=<IDLE>
213070643358494->>server:+PARSE name='', num_params=0, params_type=, query=select pg_terminate_backend(413235241)
213070643358494->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358494->>server:+DESCRIBE kind='P', name=''
213070643358494->>server:+EXECUTE name='', nb_rows=0
213070643358494->>server:+SYNC
server-->>213070643358494:-PARSE COMPLETE
server-->>213070643358494:-BIND COMPLETE
server-->>213070643358494:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='pg_terminate_backend' type=16 type_len=1 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643358494:-DATA ROW num_values=1 ---[Value 0001]--- length=1 value='t'
server-->>213070643358494:-COMMAND COMPLETE command='SELECT 1'
server-->>213070643358486:-PARSE COMPLETE
server-->>213070643358486:-BIND COMPLETE
server-->>213070643358486:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='count' type=20 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643358486:-ERROR RESPONSE Severity: 'FATAL' Message: 'terminating connection due to administrator command' Code: '57P01'
server-->>213070643358494:-READY FOR QUERY type=<IDLE>
213070643358490->>server:+DISCONNECT
213070643358494->>server:+DISCONNECT
213070643358496->>server:+SSL REQUEST
server-->>213070643358496:-SSL BACKEND ANSWER: N
213070643358496->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358496:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358496:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358496:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358496:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358496:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358496:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358496:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358496:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358496:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358496:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358496:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358496:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358496:-BACKEND KEY DATA pid=841257432, key=992978867
server-->>213070643358496:-READY FOR QUERY type=<IDLE>
213070643358496->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358496->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358496->>server:+EXECUTE name='', nb_rows=1
213070643358496->>server:+SYNC
server-->>213070643358496:-PARSE COMPLETE
server-->>213070643358496:-BIND COMPLETE
server-->>213070643358496:-COMMAND COMPLETE command='SET'
server-->>213070643358496:-READY FOR QUERY type=<IDLE>
213070643358496->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358496->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358496->>server:+EXECUTE name='', nb_rows=1
213070643358496->>server:+SYNC
server-->>213070643358496:-PARSE COMPLETE
server-->>213070643358496:-BIND COMPLETE
server-->>213070643358496:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358496:-COMMAND COMPLETE command='SET'
server-->>213070643358496:-READY FOR QUERY type=<IDLE>
213070643358496->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358496->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358496->>server:+DESCRIBE kind='P', name=''
213070643358496->>server:+EXECUTE name='', nb_rows=1
213070643358496->>server:+SYNC
server-->>213070643358496:-PARSE COMPLETE
server-->>213070643358496:-BIND COMPLETE
server-->>213070643358496:-NO DATA
server-->>213070643358496:-EMPTY QUERY RESPONSE
server-->>213070643358496:-READY FOR QUERY type=<IDLE>
213070643358496->>server:+DISCONNECT
213070643358498->>server:+SSL REQUEST
server-->>213070643358498:-SSL BACKEND ANSWER: N
213070643358498->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358498:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358498:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358498:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358498:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358498:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358498:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358498:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358498:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358498:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358498:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358498:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358498:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358498:-BACKEND KEY DATA pid=645931978, key=579078180
server-->>213070643358498:-READY FOR QUERY type=<IDLE>
213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358498->>server:+EXECUTE name='', nb_rows=1
213070643358498->>server:+SYNC
server-->>213070643358498:-PARSE COMPLETE
server-->>213070643358498:-BIND COMPLETE
server-->>213070643358498:-COMMAND COMPLETE command='SET'
server-->>213070643358498:-READY FOR QUERY type=<IDLE>
213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358498->>server:+EXECUTE name='', nb_rows=1
213070643358498->>server:+SYNC
server-->>213070643358498:-PARSE COMPLETE
server-->>213070643358498:-BIND COMPLETE
server-->>213070643358498:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358498:-COMMAND COMPLETE command='SET'
server-->>213070643358498:-READY FOR QUERY type=<IDLE>
213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358498->>server:+DESCRIBE kind='P', name=''
213070643358498->>server:+EXECUTE name='', nb_rows=1
213070643358498->>server:+SYNC
server-->>213070643358498:-PARSE COMPLETE
server-->>213070643358498:-BIND COMPLETE
server-->>213070643358498:-NO DATA
server-->>213070643358498:-EMPTY QUERY RESPONSE
server-->>213070643358498:-READY FOR QUERY type=<IDLE>
213070643358498->>server:+PARSE name='', num_params=0, params_type=, query=SELECT 1
213070643358498->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358498->>server:+DESCRIBE kind='P', name=''
213070643358498->>server:+EXECUTE name='', nb_rows=0
213070643358498->>server:+SYNC
server-->>213070643358498:-PARSE COMPLETE
server-->>213070643358498:-BIND COMPLETE
server-->>213070643358498:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='?column?' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643358498:-DATA ROW num_values=1 ---[Value 0001]--- length=1 value='1'
server-->>213070643358498:-COMMAND COMPLETE command='SELECT'
server-->>213070643358498:-READY FOR QUERY type=<IDLE>
213070643358498->>server:+DISCONNECT
213070643358500->>server:+SSL REQUEST
server-->>213070643358500:-SSL BACKEND ANSWER: N
213070643358500->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358500:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358500:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358500:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358500:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358500:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358500:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358500:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358500:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358500:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358500:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358500:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358500:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358500:-BACKEND KEY DATA pid=1632401438, key=1341645778
server-->>213070643358500:-READY FOR QUERY type=<IDLE>
213070643358500->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358500->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358500->>server:+EXECUTE name='', nb_rows=1
213070643358500->>server:+SYNC
server-->>213070643358500:-PARSE COMPLETE
server-->>213070643358500:-BIND COMPLETE
server-->>213070643358500:-COMMAND COMPLETE command='SET'
server-->>213070643358500:-READY FOR QUERY type=<IDLE>
213070643358500->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358500->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358500->>server:+EXECUTE name='', nb_rows=1
213070643358500->>server:+SYNC
server-->>213070643358500:-PARSE COMPLETE
server-->>213070643358500:-BIND COMPLETE
server-->>213070643358500:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358500:-COMMAND COMPLETE command='SET'
server-->>213070643358500:-READY FOR QUERY type=<IDLE>
213070643358500->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358500->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358500->>server:+DESCRIBE kind='P', name=''
213070643358500->>server:+EXECUTE name='', nb_rows=1
213070643358500->>server:+SYNC
server-->>213070643358500:-PARSE COMPLETE
server-->>213070643358500:-BIND COMPLETE
server-->>213070643358500:-NO DATA
server-->>213070643358500:-EMPTY QUERY RESPONSE
server-->>213070643358500:-READY FOR QUERY type=<IDLE>
213070643358500->>server:+DISCONNECT
213070643358502->>server:+SSL REQUEST
server-->>213070643358502:-SSL BACKEND ANSWER: N
213070643358502->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643358502:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643358502:-PARAMETER STATUS name='application_name', value=''
server-->>213070643358502:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643358502:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643358502:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643358502:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643358502:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643358502:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643358502:-PARAMETER STATUS name='server_version', value='13.0.0'
server-->>213070643358502:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643358502:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643358502:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643358502:-BACKEND KEY DATA pid=232586748, key=226557416
server-->>213070643358502:-READY FOR QUERY type=<IDLE>
213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358502->>server:+EXECUTE name='', nb_rows=1
213070643358502->>server:+SYNC
server-->>213070643358502:-PARSE COMPLETE
server-->>213070643358502:-BIND COMPLETE
server-->>213070643358502:-COMMAND COMPLETE command='SET'
server-->>213070643358502:-READY FOR QUERY type=<IDLE>
213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358502->>server:+EXECUTE name='', nb_rows=1
213070643358502->>server:+SYNC
server-->>213070643358502:-PARSE COMPLETE
server-->>213070643358502:-BIND COMPLETE
server-->>213070643358502:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643358502:-COMMAND COMPLETE command='SET'
server-->>213070643358502:-READY FOR QUERY type=<IDLE>
213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=
213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358502->>server:+DESCRIBE kind='P', name=''
213070643358502->>server:+EXECUTE name='', nb_rows=1
213070643358502->>server:+SYNC
server-->>213070643358502:-PARSE COMPLETE
server-->>213070643358502:-BIND COMPLETE
server-->>213070643358502:-NO DATA
server-->>213070643358502:-EMPTY QUERY RESPONSE
server-->>213070643358502:-READY FOR QUERY type=<IDLE>
213070643358502->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
213070643358502->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643358502->>server:+DESCRIBE kind='P', name=''
213070643358502->>server:+EXECUTE name='', nb_rows=0
213070643358502->>server:+SYNC
server-->>213070643358502:-PARSE COMPLETE
server-->>213070643358502:-BIND COMPLETE
server-->>213070643358502:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643358502:-DATA ROW num_values=1 ---[Value 0001]--- length=27 value='PostgresSQL 13.0 (molecula)'
server-->>213070643358502:-COMMAND COMPLETE command='SELECT'
server-->>213070643358502:-READY FOR QUERY type=<IDLE>
213070643358502->>server:+DISCONNECT
```

677
pg/lookerToPostgres.md Normal file
View file

@ -0,0 +1,677 @@
```mermaid
sequenceDiagram
participant 213070643360888 as c0
participant 213070643360892 as c1
participant 213070643360896 as c2
participant 213070643360900 as c3
participant 213070643360904 as c4
participant 213070643360908 as c5
participant 213070643360912 as c6
participant 213070643360916 as c7
participant 213070643360920 as c8
participant 213070643360924 as c9
participant 213070643360928 as c10
participant 213070643360932 as c11
213070643360888->>server:+SSL REQUEST
server-->>213070643360888:-SSL BACKEND ANSWER: N
213070643360888->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360888:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='affbc4bf')
213070643360888->>server:+PASSWORD MESSAGE password=md5de1ae46649b137ee14e14d8fd5fc6cb6
server-->>213070643360888:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360888:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360888:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360888:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360888:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360888:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360888:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360888:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360888:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360888:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360888:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360888:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360888:-BACKEND KEY DATA pid=97, key=2120775944
server-->>213070643360888:-READY FOR QUERY type=<IDLE>
213070643360888->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360888->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360888->>server:+EXECUTE name='', nb_rows=1
213070643360888->>server:+SYNC
server-->>213070643360888:-PARSE COMPLETE
server-->>213070643360888:-BIND COMPLETE
server-->>213070643360888:-COMMAND COMPLETE command='SET'
server-->>213070643360888:-READY FOR QUERY type=<IDLE>
213070643360888->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360888->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360888->>server:+EXECUTE name='', nb_rows=1
213070643360888->>server:+SYNC
server-->>213070643360888:-PARSE COMPLETE
server-->>213070643360888:-BIND COMPLETE
server-->>213070643360888:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360888:-COMMAND COMPLETE command='SET'
server-->>213070643360888:-READY FOR QUERY type=<IDLE>
213070643360888->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360888->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360888->>server:+DESCRIBE kind='P', name=''
213070643360888->>server:+EXECUTE name='', nb_rows=1
213070643360888->>server:+SYNC
server-->>213070643360888:-PARSE COMPLETE
server-->>213070643360888:-BIND COMPLETE
server-->>213070643360888:-NO DATA
server-->>213070643360888:-EMPTY QUERY RESPONSE
server-->>213070643360888:-READY FOR QUERY type=<IDLE>
213070643360888->>server:+DISCONNECT
213070643360892->>server:+SSL REQUEST
server-->>213070643360892:-SSL BACKEND ANSWER: N
213070643360892->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360892:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='0b76ce86')
213070643360892->>server:+PASSWORD MESSAGE password=md5b82e4b6283fe5694c0199ca058378bb8
server-->>213070643360892:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360892:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360892:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360892:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360892:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360892:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360892:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360892:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360892:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360892:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360892:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360892:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360892:-BACKEND KEY DATA pid=98, key=1329847468
server-->>213070643360892:-READY FOR QUERY type=<IDLE>
213070643360892->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360892->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360892->>server:+EXECUTE name='', nb_rows=1
213070643360892->>server:+SYNC
server-->>213070643360892:-PARSE COMPLETE
server-->>213070643360892:-BIND COMPLETE
server-->>213070643360892:-COMMAND COMPLETE command='SET'
server-->>213070643360892:-READY FOR QUERY type=<IDLE>
213070643360892->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360892->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360892->>server:+EXECUTE name='', nb_rows=1
213070643360892->>server:+SYNC
server-->>213070643360892:-PARSE COMPLETE
server-->>213070643360892:-BIND COMPLETE
server-->>213070643360892:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360892:-COMMAND COMPLETE command='SET'
server-->>213070643360892:-READY FOR QUERY type=<IDLE>
213070643360892->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360892->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360892->>server:+DESCRIBE kind='P', name=''
213070643360892->>server:+EXECUTE name='', nb_rows=1
213070643360892->>server:+SYNC
server-->>213070643360892:-PARSE COMPLETE
server-->>213070643360892:-BIND COMPLETE
server-->>213070643360892:-NO DATA
server-->>213070643360892:-EMPTY QUERY RESPONSE
server-->>213070643360892:-READY FOR QUERY type=<IDLE>
213070643360892->>server:+DISCONNECT
213070643360896->>server:+SSL REQUEST
server-->>213070643360896:-SSL BACKEND ANSWER: N
213070643360896->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360896:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='80e88231')
213070643360896->>server:+PASSWORD MESSAGE password=md59023db05ad8c94976359641cf0ada45a
server-->>213070643360896:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360896:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360896:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360896:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360896:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360896:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360896:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360896:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360896:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360896:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360896:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360896:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360896:-BACKEND KEY DATA pid=99, key=1084480878
server-->>213070643360896:-READY FOR QUERY type=<IDLE>
213070643360896->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360896->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360896->>server:+EXECUTE name='', nb_rows=1
213070643360896->>server:+SYNC
server-->>213070643360896:-PARSE COMPLETE
server-->>213070643360896:-BIND COMPLETE
server-->>213070643360896:-COMMAND COMPLETE command='SET'
server-->>213070643360896:-READY FOR QUERY type=<IDLE>
213070643360896->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360896->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360896->>server:+EXECUTE name='', nb_rows=1
213070643360896->>server:+SYNC
server-->>213070643360896:-PARSE COMPLETE
server-->>213070643360896:-BIND COMPLETE
server-->>213070643360896:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360896:-COMMAND COMPLETE command='SET'
server-->>213070643360896:-READY FOR QUERY type=<IDLE>
213070643360896->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360896->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360896->>server:+DESCRIBE kind='P', name=''
213070643360896->>server:+EXECUTE name='', nb_rows=1
213070643360896->>server:+SYNC
server-->>213070643360896:-PARSE COMPLETE
server-->>213070643360896:-BIND COMPLETE
server-->>213070643360896:-NO DATA
server-->>213070643360896:-EMPTY QUERY RESPONSE
server-->>213070643360896:-READY FOR QUERY type=<IDLE>
213070643360896->>server:+DISCONNECT
213070643360900->>server:+SSL REQUEST
server-->>213070643360900:-SSL BACKEND ANSWER: N
213070643360900->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360900:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='68a07783')
213070643360900->>server:+PASSWORD MESSAGE password=md5136c8b8ec47347f93827ec5d7023199c
server-->>213070643360900:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360900:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360900:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360900:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360900:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360900:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360900:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360900:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360900:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360900:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360900:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360900:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360900:-BACKEND KEY DATA pid=100, key=594556468
server-->>213070643360900:-READY FOR QUERY type=<IDLE>
213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360900->>server:+EXECUTE name='', nb_rows=1
213070643360900->>server:+SYNC
server-->>213070643360900:-PARSE COMPLETE
server-->>213070643360900:-BIND COMPLETE
server-->>213070643360900:-COMMAND COMPLETE command='SET'
server-->>213070643360900:-READY FOR QUERY type=<IDLE>
213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360900->>server:+EXECUTE name='', nb_rows=1
213070643360900->>server:+SYNC
server-->>213070643360900:-PARSE COMPLETE
server-->>213070643360900:-BIND COMPLETE
server-->>213070643360900:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360900:-COMMAND COMPLETE command='SET'
server-->>213070643360900:-READY FOR QUERY type=<IDLE>
213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360900->>server:+DESCRIBE kind='P', name=''
213070643360900->>server:+EXECUTE name='', nb_rows=1
213070643360900->>server:+SYNC
server-->>213070643360900:-PARSE COMPLETE
server-->>213070643360900:-BIND COMPLETE
server-->>213070643360900:-NO DATA
server-->>213070643360900:-EMPTY QUERY RESPONSE
server-->>213070643360900:-READY FOR QUERY type=<IDLE>
213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SELECT pg_backend_pid()
213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360900->>server:+DESCRIBE kind='P', name=''
213070643360900->>server:+EXECUTE name='', nb_rows=0
213070643360900->>server:+SYNC
server-->>213070643360900:-PARSE COMPLETE
server-->>213070643360900:-BIND COMPLETE
server-->>213070643360900:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='pg_backend_pid' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643360900:-DATA ROW num_values=1 ---[Value 0001]--- length=3 value='100'
server-->>213070643360900:-COMMAND COMPLETE command='SELECT 1'
server-->>213070643360900:-READY FOR QUERY type=<IDLE>
213070643360900->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360900->>server:+DESCRIBE kind='P', name=''
213070643360900->>server:+EXECUTE name='', nb_rows=0
213070643360900->>server:+SYNC
server-->>213070643360900:-PARSE COMPLETE
server-->>213070643360900:-BIND COMPLETE
server-->>213070643360900:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643360900:-DATA ROW num_values=1 ---[Value 0001]--- length=112 value='PostgreSQL 13.3 (Debian 13.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit'
server-->>213070643360900:-COMMAND COMPLETE command='SELECT 1'
server-->>213070643360900:-READY FOR QUERY type=<IDLE>
213070643360900->>server:+PARSE name='', num_params=0, params_type=, query= SELECT COUNT(*) FROM pg_type AS t0, pg_aggregate AS t1, pg_settings AS t2, pg_settings AS t3
213070643360900->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360900->>server:+DESCRIBE kind='P', name=''
213070643360900->>server:+EXECUTE name='', nb_rows=0
213070643360900->>server:+SYNC
213070643360904->>server:+SSL REQUEST
server-->>213070643360904:-SSL BACKEND ANSWER: N
213070643360904->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360904:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='a03ee692')
213070643360904->>server:+PASSWORD MESSAGE password=md5cfedb52ae0cafa87a4f1066df3ba6802
server-->>213070643360904:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360904:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360904:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360904:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360904:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360904:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360904:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360904:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360904:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360904:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360904:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360904:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360904:-BACKEND KEY DATA pid=101, key=2854304053
server-->>213070643360904:-READY FOR QUERY type=<IDLE>
213070643360904->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360904->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360904->>server:+EXECUTE name='', nb_rows=1
213070643360904->>server:+SYNC
server-->>213070643360904:-PARSE COMPLETE
server-->>213070643360904:-BIND COMPLETE
server-->>213070643360904:-COMMAND COMPLETE command='SET'
server-->>213070643360904:-READY FOR QUERY type=<IDLE>
213070643360904->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360904->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360904->>server:+EXECUTE name='', nb_rows=1
213070643360904->>server:+SYNC
server-->>213070643360904:-PARSE COMPLETE
server-->>213070643360904:-BIND COMPLETE
server-->>213070643360904:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360904:-COMMAND COMPLETE command='SET'
server-->>213070643360904:-READY FOR QUERY type=<IDLE>
213070643360904->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360904->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360904->>server:+DESCRIBE kind='P', name=''
213070643360904->>server:+EXECUTE name='', nb_rows=1
213070643360904->>server:+SYNC
server-->>213070643360904:-PARSE COMPLETE
server-->>213070643360904:-BIND COMPLETE
server-->>213070643360904:-NO DATA
server-->>213070643360904:-EMPTY QUERY RESPONSE
server-->>213070643360904:-READY FOR QUERY type=<IDLE>
213070643360904->>server:+DISCONNECT
213070643360908->>server:+SSL REQUEST
server-->>213070643360908:-SSL BACKEND ANSWER: N
213070643360908->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360908:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='be3afb9d')
213070643360908->>server:+PASSWORD MESSAGE password=md57b5cfa30f89fc7733c814addcd548c03
server-->>213070643360908:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360908:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360908:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360908:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360908:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360908:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360908:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360908:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360908:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360908:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360908:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360908:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360908:-BACKEND KEY DATA pid=102, key=3089396074
server-->>213070643360908:-READY FOR QUERY type=<IDLE>
213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360908->>server:+EXECUTE name='', nb_rows=1
213070643360908->>server:+SYNC
server-->>213070643360908:-PARSE COMPLETE
server-->>213070643360908:-BIND COMPLETE
server-->>213070643360908:-COMMAND COMPLETE command='SET'
server-->>213070643360908:-READY FOR QUERY type=<IDLE>
213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360908->>server:+EXECUTE name='', nb_rows=1
213070643360908->>server:+SYNC
server-->>213070643360908:-PARSE COMPLETE
server-->>213070643360908:-BIND COMPLETE
server-->>213070643360908:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360908:-COMMAND COMPLETE command='SET'
server-->>213070643360908:-READY FOR QUERY type=<IDLE>
213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360908->>server:+DESCRIBE kind='P', name=''
213070643360908->>server:+EXECUTE name='', nb_rows=1
213070643360908->>server:+SYNC
server-->>213070643360908:-PARSE COMPLETE
server-->>213070643360908:-BIND COMPLETE
server-->>213070643360908:-NO DATA
server-->>213070643360908:-EMPTY QUERY RESPONSE
server-->>213070643360908:-READY FOR QUERY type=<IDLE>
213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360908->>server:+DESCRIBE kind='P', name=''
213070643360908->>server:+EXECUTE name='', nb_rows=0
213070643360908->>server:+SYNC
server-->>213070643360908:-PARSE COMPLETE
server-->>213070643360908:-BIND COMPLETE
server-->>213070643360908:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643360908:-DATA ROW num_values=1 ---[Value 0001]--- length=112 value='PostgreSQL 13.3 (Debian 13.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit'
server-->>213070643360908:-COMMAND COMPLETE command='SELECT 1'
server-->>213070643360908:-READY FOR QUERY type=<IDLE>
213070643360908->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360908->>server:+DESCRIBE kind='P', name=''
213070643360908->>server:+EXECUTE name='', nb_rows=1
213070643360908->>server:+SYNC
server-->>213070643360908:-PARSE COMPLETE
server-->>213070643360908:-BIND COMPLETE
server-->>213070643360908:-NO DATA
server-->>213070643360908:-EMPTY QUERY RESPONSE
server-->>213070643360908:-READY FOR QUERY type=<IDLE>
213070643360908->>server:+PARSE name='', num_params=0, params_type=, query= SELECT pid as id, query as stmt, EXTRACT(seconds from query_start - NOW()) as elapsed_time FROM pg_stat_activity WHERE usename='docker'
213070643360908->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360908->>server:+DESCRIBE kind='P', name=''
213070643360908->>server:+EXECUTE name='', nb_rows=0
213070643360908->>server:+SYNC
server-->>213070643360908:-PARSE COMPLETE
server-->>213070643360908:-BIND COMPLETE
server-->>213070643360908:-ROW DESCRIPTION: num_fields=3 ---[Field 01]--- name='id' type=23 type_len=4 type_mod=4294967295 relid=12250 attnum=3 format=0 ---[Field 02]--- name='stmt' type=25 type_len=65535 type_mod=4294967295 relid=12250 attnum=20 format=0 ---[Field 03]--- name='elapsed_time' type=701 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643360908:-DATA ROW num_values=3 ---[Value 0001]--- length=2 value='81' ---[Value 0002]--- length=0 value='' ---[Value 0003]--- length=-1 value=NULL
server-->>213070643360908:-DATA ROW num_values=3 ---[Value 0001]--- length=3 value='100' ---[Value 0002]--- length=148 value=' SELECT COUNT(*). FROM pg_type AS t0,. pg_aggregate AS t1,. pg_settings AS t2,. pg_settings AS t3.' ---[Value 0003]--- length=9 value='-1.155449'
server-->>213070643360908:-DATA ROW num_values=3 ---[Value 0001]--- length=3 value='102' ---[Value 0002]--- length=190 value=' SELECT pid as id,. query as stmt,. EXTRACT(seconds from query_start - NOW()) as elapsed_time. FROM pg_stat_activity. WHERE usename='docker'.' ---[Value 0003]--- length=8 value='0.002903'
server-->>213070643360908:-COMMAND COMPLETE command='SELECT 3'
server-->>213070643360908:-READY FOR QUERY type=<IDLE>
213070643360912->>server:+SSL REQUEST
server-->>213070643360912:-SSL BACKEND ANSWER: N
213070643360912->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360912:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='14444412')
213070643360912->>server:+PASSWORD MESSAGE password=md59cef18c88b7988d7ac2f215fc1569c62
server-->>213070643360912:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360912:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360912:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360912:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360912:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360912:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360912:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360912:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360912:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360912:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360912:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360912:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360912:-BACKEND KEY DATA pid=103, key=1911102642
server-->>213070643360912:-READY FOR QUERY type=<IDLE>
213070643360912->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360912->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360912->>server:+EXECUTE name='', nb_rows=1
213070643360912->>server:+SYNC
server-->>213070643360912:-PARSE COMPLETE
server-->>213070643360912:-BIND COMPLETE
server-->>213070643360912:-COMMAND COMPLETE command='SET'
server-->>213070643360912:-READY FOR QUERY type=<IDLE>
213070643360912->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360912->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360912->>server:+EXECUTE name='', nb_rows=1
213070643360912->>server:+SYNC
server-->>213070643360912:-PARSE COMPLETE
server-->>213070643360912:-BIND COMPLETE
server-->>213070643360912:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360912:-COMMAND COMPLETE command='SET'
server-->>213070643360912:-READY FOR QUERY type=<IDLE>
213070643360912->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360912->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360912->>server:+DESCRIBE kind='P', name=''
213070643360912->>server:+EXECUTE name='', nb_rows=1
213070643360912->>server:+SYNC
server-->>213070643360912:-PARSE COMPLETE
server-->>213070643360912:-BIND COMPLETE
server-->>213070643360912:-NO DATA
server-->>213070643360912:-EMPTY QUERY RESPONSE
server-->>213070643360912:-READY FOR QUERY type=<IDLE>
213070643360912->>server:+DISCONNECT
213070643360916->>server:+SSL REQUEST
server-->>213070643360916:-SSL BACKEND ANSWER: N
213070643360916->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360916:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='9eb7546f')
213070643360916->>server:+PASSWORD MESSAGE password=md5c7655226306ab14fa17e44dba876a348
server-->>213070643360916:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360916:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360916:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360916:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360916:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360916:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360916:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360916:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360916:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360916:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360916:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360916:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360916:-BACKEND KEY DATA pid=104, key=927987783
server-->>213070643360916:-READY FOR QUERY type=<IDLE>
213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360916->>server:+EXECUTE name='', nb_rows=1
213070643360916->>server:+SYNC
server-->>213070643360916:-PARSE COMPLETE
server-->>213070643360916:-BIND COMPLETE
server-->>213070643360916:-COMMAND COMPLETE command='SET'
server-->>213070643360916:-READY FOR QUERY type=<IDLE>
213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360916->>server:+EXECUTE name='', nb_rows=1
213070643360916->>server:+SYNC
server-->>213070643360916:-PARSE COMPLETE
server-->>213070643360916:-BIND COMPLETE
server-->>213070643360916:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360916:-COMMAND COMPLETE command='SET'
server-->>213070643360916:-READY FOR QUERY type=<IDLE>
213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360916->>server:+DESCRIBE kind='P', name=''
213070643360916->>server:+EXECUTE name='', nb_rows=1
213070643360916->>server:+SYNC
server-->>213070643360916:-PARSE COMPLETE
server-->>213070643360916:-BIND COMPLETE
server-->>213070643360916:-NO DATA
server-->>213070643360916:-EMPTY QUERY RESPONSE
server-->>213070643360916:-READY FOR QUERY type=<IDLE>
213070643360916->>server:+PARSE name='', num_params=0, params_type=, query=select pg_terminate_backend(100)
213070643360916->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360916->>server:+DESCRIBE kind='P', name=''
213070643360916->>server:+EXECUTE name='', nb_rows=0
213070643360916->>server:+SYNC
server-->>213070643360916:-PARSE COMPLETE
server-->>213070643360916:-BIND COMPLETE
server-->>213070643360916:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='pg_terminate_backend' type=16 type_len=1 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643360916:-DATA ROW num_values=1 ---[Value 0001]--- length=1 value='t'
server-->>213070643360916:-COMMAND COMPLETE command='SELECT 1'
server-->>213070643360916:-READY FOR QUERY type=<IDLE>
server-->>213070643360900:-PARSE COMPLETE
server-->>213070643360900:-BIND COMPLETE
server-->>213070643360900:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='count' type=20 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643360900:-ERROR RESPONSE File: 'postgres.c' Severity: 'FATAL' Message: 'terminating connection due to administrator command' Code: '57P01' Routine: 'ProcessInterrupts' Line: '3090'
213070643360908->>server:+DISCONNECT
213070643360916->>server:+DISCONNECT
213070643360920->>server:+SSL REQUEST
server-->>213070643360920:-SSL BACKEND ANSWER: N
213070643360920->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360920:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='1ad4dcf7')
213070643360920->>server:+PASSWORD MESSAGE password=md554de511c5219f67a8bbe92da445fa5a2
server-->>213070643360920:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360920:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360920:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360920:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360920:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360920:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360920:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360920:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360920:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360920:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360920:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360920:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360920:-BACKEND KEY DATA pid=105, key=3630332440
server-->>213070643360920:-READY FOR QUERY type=<IDLE>
213070643360920->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360920->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360920->>server:+EXECUTE name='', nb_rows=1
213070643360920->>server:+SYNC
server-->>213070643360920:-PARSE COMPLETE
server-->>213070643360920:-BIND COMPLETE
server-->>213070643360920:-COMMAND COMPLETE command='SET'
server-->>213070643360920:-READY FOR QUERY type=<IDLE>
213070643360920->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360920->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360920->>server:+EXECUTE name='', nb_rows=1
213070643360920->>server:+SYNC
server-->>213070643360920:-PARSE COMPLETE
server-->>213070643360920:-BIND COMPLETE
server-->>213070643360920:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360920:-COMMAND COMPLETE command='SET'
server-->>213070643360920:-READY FOR QUERY type=<IDLE>
213070643360920->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360920->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360920->>server:+DESCRIBE kind='P', name=''
213070643360920->>server:+EXECUTE name='', nb_rows=1
213070643360920->>server:+SYNC
server-->>213070643360920:-PARSE COMPLETE
server-->>213070643360920:-BIND COMPLETE
server-->>213070643360920:-NO DATA
server-->>213070643360920:-EMPTY QUERY RESPONSE
server-->>213070643360920:-READY FOR QUERY type=<IDLE>
213070643360920->>server:+DISCONNECT
213070643360924->>server:+SSL REQUEST
server-->>213070643360924:-SSL BACKEND ANSWER: N
213070643360924->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360924:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='494c352f')
213070643360924->>server:+PASSWORD MESSAGE password=md5919fc9ed056904fa0d8e1dd00625a556
server-->>213070643360924:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360924:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360924:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360924:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360924:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360924:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360924:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360924:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360924:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360924:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360924:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360924:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360924:-BACKEND KEY DATA pid=106, key=3364308023
server-->>213070643360924:-READY FOR QUERY type=<IDLE>
213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360924->>server:+EXECUTE name='', nb_rows=1
213070643360924->>server:+SYNC
server-->>213070643360924:-PARSE COMPLETE
server-->>213070643360924:-BIND COMPLETE
server-->>213070643360924:-COMMAND COMPLETE command='SET'
server-->>213070643360924:-READY FOR QUERY type=<IDLE>
213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360924->>server:+EXECUTE name='', nb_rows=1
213070643360924->>server:+SYNC
server-->>213070643360924:-PARSE COMPLETE
server-->>213070643360924:-BIND COMPLETE
server-->>213070643360924:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360924:-COMMAND COMPLETE command='SET'
server-->>213070643360924:-READY FOR QUERY type=<IDLE>
213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360924->>server:+DESCRIBE kind='P', name=''
213070643360924->>server:+EXECUTE name='', nb_rows=1
213070643360924->>server:+SYNC
server-->>213070643360924:-PARSE COMPLETE
server-->>213070643360924:-BIND COMPLETE
server-->>213070643360924:-NO DATA
server-->>213070643360924:-EMPTY QUERY RESPONSE
server-->>213070643360924:-READY FOR QUERY type=<IDLE>
213070643360924->>server:+PARSE name='', num_params=0, params_type=, query=SELECT 1
213070643360924->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360924->>server:+DESCRIBE kind='P', name=''
213070643360924->>server:+EXECUTE name='', nb_rows=0
213070643360924->>server:+SYNC
server-->>213070643360924:-PARSE COMPLETE
server-->>213070643360924:-BIND COMPLETE
server-->>213070643360924:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='?column?' type=23 type_len=4 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643360924:-DATA ROW num_values=1 ---[Value 0001]--- length=1 value='1'
server-->>213070643360924:-COMMAND COMPLETE command='SELECT 1'
server-->>213070643360924:-READY FOR QUERY type=<IDLE>
213070643360924->>server:+DISCONNECT
213070643360928->>server:+SSL REQUEST
server-->>213070643360928:-SSL BACKEND ANSWER: N
213070643360928->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360928:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='4b2173db')
213070643360928->>server:+PASSWORD MESSAGE password=md53977c39c5c7cdef7f80e74b256a8ce25
server-->>213070643360928:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360928:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360928:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360928:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360928:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360928:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360928:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360928:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360928:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360928:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360928:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360928:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360928:-BACKEND KEY DATA pid=107, key=1988416582
server-->>213070643360928:-READY FOR QUERY type=<IDLE>
213070643360928->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360928->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360928->>server:+EXECUTE name='', nb_rows=1
213070643360928->>server:+SYNC
server-->>213070643360928:-PARSE COMPLETE
server-->>213070643360928:-BIND COMPLETE
server-->>213070643360928:-COMMAND COMPLETE command='SET'
server-->>213070643360928:-READY FOR QUERY type=<IDLE>
213070643360928->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360928->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360928->>server:+EXECUTE name='', nb_rows=1
213070643360928->>server:+SYNC
server-->>213070643360928:-PARSE COMPLETE
server-->>213070643360928:-BIND COMPLETE
server-->>213070643360928:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360928:-COMMAND COMPLETE command='SET'
server-->>213070643360928:-READY FOR QUERY type=<IDLE>
213070643360928->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360928->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360928->>server:+DESCRIBE kind='P', name=''
213070643360928->>server:+EXECUTE name='', nb_rows=1
213070643360928->>server:+SYNC
server-->>213070643360928:-PARSE COMPLETE
server-->>213070643360928:-BIND COMPLETE
server-->>213070643360928:-NO DATA
server-->>213070643360928:-EMPTY QUERY RESPONSE
server-->>213070643360928:-READY FOR QUERY type=<IDLE>
213070643360928->>server:+DISCONNECT
213070643360932->>server:+SSL REQUEST
server-->>213070643360932:-SSL BACKEND ANSWER: N
213070643360932->>server:+STARTUP MESSAGE version: 3 database=trait_store extra_float_digits=2 TimeZone=GMT client_encoding=UTF8 user=docker DateStyle=ISO
server-->>213070643360932:-AUTHENTIFICATION REQUEST code=5 (MD5 salt='f6d0f51d')
213070643360932->>server:+PASSWORD MESSAGE password=md5b5b322fe9bcc7c242c6b44cc8a7898e8
server-->>213070643360932:-AUTHENTIFICATION REQUEST code=0 (SUCCESS)
server-->>213070643360932:-PARAMETER STATUS name='application_name', value=''
server-->>213070643360932:-PARAMETER STATUS name='client_encoding', value='UTF8'
server-->>213070643360932:-PARAMETER STATUS name='DateStyle', value='ISO, MDY'
server-->>213070643360932:-PARAMETER STATUS name='integer_datetimes', value='on'
server-->>213070643360932:-PARAMETER STATUS name='IntervalStyle', value='postgres'
server-->>213070643360932:-PARAMETER STATUS name='is_superuser', value='on'
server-->>213070643360932:-PARAMETER STATUS name='server_encoding', value='UTF8'
server-->>213070643360932:-PARAMETER STATUS name='server_version', value='13.3 (Debian 13.3-1.pgdg100+1)'
server-->>213070643360932:-PARAMETER STATUS name='session_authorization', value='docker'
server-->>213070643360932:-PARAMETER STATUS name='standard_conforming_strings', value='on'
server-->>213070643360932:-PARAMETER STATUS name='TimeZone', value='GMT'
server-->>213070643360932:-BACKEND KEY DATA pid=108, key=997991029
server-->>213070643360932:-READY FOR QUERY type=<IDLE>
213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=SET extra_float_digits = 3
213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360932->>server:+EXECUTE name='', nb_rows=1
213070643360932->>server:+SYNC
server-->>213070643360932:-PARSE COMPLETE
server-->>213070643360932:-BIND COMPLETE
server-->>213070643360932:-COMMAND COMPLETE command='SET'
server-->>213070643360932:-READY FOR QUERY type=<IDLE>
213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=SET application_name = 'PostgreSQL JDBC Driver'
213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360932->>server:+EXECUTE name='', nb_rows=1
213070643360932->>server:+SYNC
server-->>213070643360932:-PARSE COMPLETE
server-->>213070643360932:-BIND COMPLETE
server-->>213070643360932:-PARAMETER STATUS name='application_name', value='PostgreSQL JDBC Driver'
server-->>213070643360932:-COMMAND COMPLETE command='SET'
server-->>213070643360932:-READY FOR QUERY type=<IDLE>
213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=
213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360932->>server:+DESCRIBE kind='P', name=''
213070643360932->>server:+EXECUTE name='', nb_rows=1
213070643360932->>server:+SYNC
server-->>213070643360932:-PARSE COMPLETE
server-->>213070643360932:-BIND COMPLETE
server-->>213070643360932:-NO DATA
server-->>213070643360932:-EMPTY QUERY RESPONSE
server-->>213070643360932:-READY FOR QUERY type=<IDLE>
213070643360932->>server:+PARSE name='', num_params=0, params_type=, query=SELECT VERSION() AS version
213070643360932->>server:+BIND portal='', name='', num_formats=0, formats=, num_params=0, params=
213070643360932->>server:+DESCRIBE kind='P', name=''
213070643360932->>server:+EXECUTE name='', nb_rows=0
213070643360932->>server:+SYNC
server-->>213070643360932:-PARSE COMPLETE
server-->>213070643360932:-BIND COMPLETE
server-->>213070643360932:-ROW DESCRIPTION: num_fields=1 ---[Field 01]--- name='version' type=25 type_len=65535 type_mod=4294967295 relid=0 attnum=0 format=0
server-->>213070643360932:-DATA ROW num_values=1 ---[Value 0001]--- length=112 value='PostgreSQL 13.3 (Debian 13.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit'
server-->>213070643360932:-COMMAND COMPLETE command='SELECT 1'
server-->>213070643360932:-READY FOR QUERY type=<IDLE>
213070643360932->>server:+DISCONNECT
```

183
pg/message/message.go
View file

@ -17,6 +17,7 @@ package message
import (
"bytes"
"encoding/binary"
"fmt"
)
// Type is a byte indicating the type of a Postgres message.
@ -29,11 +30,11 @@ const (
// TypeReadyForQuery is a message used to indicate that the server is ready for another query.
TypeReadyForQuery Type = 'Z'
// TypeCommandComplete is a message used to indicate that a query has completed.
// TypeCommandComplete is a Backend message used to indicate that a query has completed.
TypeCommandComplete Type = 'C'
// TypeError is an error message.
TypeError Type = 'E'
// TypeClos is a message used to indicate that a query has completed. Frontend
TypeClose Type = 'C'
// TypeRowDescription is a message indicating the column types of the result rows from a query.
TypeRowDescription Type = 'T'
@ -52,6 +53,20 @@ const (
// TypeBackendKeyData contains a cancellation key for the client to use later.
TypeBackendKeyData Type = 'K'
TypeParse Type = 'P'
TypeParseComplete Type = '1'
TypeBind Type = 'B'
TypeBindComplete Type = '2'
TypeExecute Type = 'E' // Frontend
TypeError Type = 'E' // Backend
TypeSync Type = 'S' // Frontend
TypeParameterStatus Type = 'S' // Backend
TypeDescribe Type = 'D' // Frontend
TypeNoData Type = 'n' // backend
TypeEmptyQueryResponse Type = 'I' // backend
)
// AuthenticationOK is a message indicating that authentication has completed.
@ -59,6 +74,22 @@ var AuthenticationOK = Message{
Type: TypeAuthentication,
Data: []byte{0, 0, 0, 0},
}
var ParseOK = Message{
Type: TypeParseComplete,
Data: []byte{},
}
var BindComplete = Message{
Type: TypeBindComplete,
Data: []byte{},
}
var NoData = Message{
Type: TypeNoData,
Data: []byte{},
}
var EmptyQueryResponse = Message{
Type: TypeEmptyQueryResponse,
Data: []byte{},
}
// Message is a Postgres message value.
type Message struct {
@ -66,6 +97,49 @@ type Message struct {
Data []byte
}
//debug tools
func viewString(b []byte) string {
r := []rune(string(b))
for i := range r {
if r[i] < 32 || r[i] > 126 {
r[i] = '.'
}
}
return string(r)
}
func min(a, b int) int {
if a < b {
return a
}
return b
}
func (m *Message) Dump(prefix string) {
n := len(m.Data)
rowcount := 0
stop := (n / 8) * 8
k := 0
fmt.Printf("\n %s type: '%c'\n", prefix, m.Type)
for i := 0; i <= stop; i += 8 {
k++
if i+8 < n {
rowcount = 8
} else {
rowcount = min(k*8, n) % 8
}
fmt.Printf("pos %02d hex: ", i)
for j := 0; j < rowcount; j++ {
fmt.Printf("%02x ", m.Data[i+j])
}
for j := rowcount; j < 8; j++ {
fmt.Printf(" ")
}
fmt.Printf(" '%s'\n", viewString(m.Data[i:(i+rowcount)]))
}
}
//endTools
// TransactionStatus is the current transaction state.
type TransactionStatus byte
@ -98,6 +172,14 @@ func (e *Encoder) i32(i int32) error {
return err
}
/* removed for linter now
func (e *Encoder) u32(i uint32) error {
binary.BigEndian.PutUint32(e.scratch[:], i)
_, err := e.buf.Write(e.scratch[:])
return err
}
*/
// ReadyForQuery encodes a "ready for query" message.
func (e *Encoder) ReadyForQuery(status TransactionStatus) (Message, error) {
e.buf.Reset()
@ -149,6 +231,9 @@ const (
// NoticeFieldHint is a suggestion of how to address the issue.
NoticeFieldHint NoticeFieldType = 'H'
// NoticeFieldHint the SQLSTATE code for the error (see Appendix A). Not localizable. Always present.
NoticeFieldCode NoticeFieldType = 'C'
)
// NoticeField is a field in an error or notice.
@ -365,3 +450,95 @@ func (e *Encoder) BackendKeyData(pid, key int32) (Message, error) {
Data: e.buf.Bytes(),
}, nil
}
func (e *Encoder) ParameterStatus(param, value string) (Message, error) {
e.buf.Reset()
//param + NULL + value+ NULL
_, err := e.buf.WriteString(param)
if err != nil {
return Message{}, err
}
err = e.buf.WriteByte(0)
if err != nil {
return Message{}, err
}
_, err = e.buf.WriteString(value)
if err != nil {
return Message{}, err
}
err = e.buf.WriteByte(0)
if err != nil {
return Message{}, err
}
return Message{
Type: TypeParameterStatus,
Data: e.buf.Bytes(),
}, nil
}
type SimpleColumn struct {
Name string
Typeid int32
Typelen int16
}
func (e *Encoder) EncodeColumn(name string, typeid int32, typelen int16) (Message, error) {
return e.EncodeColumns(SimpleColumn{
Name: name,
Typeid: typeid,
Typelen: typelen,
})
}
func (e *Encoder) EncodeColumns(cols ...SimpleColumn) (Message, error) {
e.buf.Reset()
err := e.i16(int16(len(cols))) // number of columns in result
if err != nil {
return Message{}, nil
}
for _, col := range cols {
_, err = e.buf.WriteString(col.Name) // column name
if err != nil {
return Message{}, err
}
err = e.buf.WriteByte(0) //null terminate
if err != nil {
return Message{}, err
}
err = e.i32(0) //tabel id
if err != nil {
return Message{}, err
}
err = e.i16(0) //field id(attnum)
if err != nil {
return Message{}, err
}
err = e.i32(col.Typeid) //type_id
if err != nil {
return Message{}, err
}
err = e.i16(col.Typelen) //type_len
if err != nil {
return Message{}, err
}
err = e.i32(-1) //type_mod
if err != nil {
return Message{}, err
}
err = e.i16(0) //format 0 text 1 binary
if err != nil {
return Message{}, err
}
}
return Message{
Type: TypeRowDescription,
Data: e.buf.Bytes(),
}, nil
}

6
pg/pgtest/handler.go
View file

@ -30,6 +30,12 @@ type HandlerFunc func(context.Context, pg.QueryResultWriter, pg.Query) error
func (h HandlerFunc) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
return h(ctx, w, q)
}
func (h HandlerFunc) HandleSchema(ctx context.Context, portal *pg.Portal) error {
return nil
}
func (h HandlerFunc) Version() string {
return "testv1"
}
var _ pg.QueryHandler = HandlerFunc(nil)

540
pg/protocol.go
View file

@ -25,12 +25,15 @@ import (
"io"
"io/ioutil"
"net"
"regexp"
"strings"
"sync"
"time"
"github.com/molecula/featurebase/v2/pg/message"
"github.com/molecula/featurebase/v2/sql"
"github.com/pkg/errors"
"vitess.io/vitess/go/vt/sqlparser"
)
// Protocol is a Postgres protocol version.
@ -48,6 +51,9 @@ const (
// ProtocolSupported is the main protocol version supported by this package.
ProtocolSupported Protocol = ProtocolPostgres30
// PgServerVersion is the latest version of postgres that we claim to support.
PgServerVersion = "13.0.0"
)
// Major returns the major revision of the protocol.
@ -233,6 +239,377 @@ func (s *Server) handleCancel(ctx context.Context, conn net.Conn, data []byte) e
return nil
}
func (s *Server) SendParameterStatus(w *message.WireWriter, param, value string, encoder *message.Encoder) error {
msg, err := encoder.ParameterStatus(param, value)
if err != nil {
return err
}
err = w.WriteMessage(msg)
if err != nil {
return errors.Wrap(err, "sending parameter status")
}
return nil
}
type Result struct {
}
type PgType byte
// Constants used to indicated query interception
// Only pgPassOn is allowed to be processed in Featurebase query handling
const (
pgPassOn PgType = 'x'
pgBackendPid PgType = 'a'
pgVersion PgType = 'b'
pgCountType PgType = 'c'
pgQueryTime PgType = 'd'
pgTerminate PgType = 'e'
pgEmpty PgType = 'f'
pgSetApplication PgType = 'g'
pgSelect1 PgType = 'h'
pgSchema PgType = 'i'
pgBegin PgType = 'j'
)
type Portal struct {
Name string
Writer *message.WireWriter
commands []message.Message
Encoder *message.Encoder
mapper *sql.Mapper
sql string
pgspecial PgType
pid int32
queryStart time.Time
server *Server
cancelNotify <-chan struct{}
}
func (p *Portal) Reset() {
p.Name = ""
p.sql = ""
p.pgspecial = pgPassOn
p.commands = p.commands[:0]
}
func (p *Portal) Bind() {
p.Add(message.BindComplete)
}
var lookPQL = regexp.MustCompile(`\[.*\].*\)\z`)
func (p *Portal) Parse(data []byte) {
p.queryStart = time.Now()
queryStr := string(bytes.Trim(data, "\x00"))
foundPQL := lookPQL.FindStringSubmatch(queryStr)
if len(foundPQL) > 0 {
p.sql = foundPQL[0]
p.Name = "PQL"
p.pgspecial = pgPassOn
p.Add(message.ParseOK)
return
}
if strings.Contains(queryStr, "EXTRACT") {
// had to add this hack because the vitis parser doesn't handle...
/*
SELECT pid as id,
query as stmt,
EXTRACT(seconds from query_start - NOW()) as elapsed_time
FROM pg_stat_activity
WHERE usename='docker'`
*/
p.pgspecial = pgQueryTime
p.Name = "SELECT"
p.sql = queryStr
p.Add(message.ParseOK)
return
}
if len(queryStr) > 2 {
query, err := p.mapper.MapSQL(queryStr)
if err != nil {
return
}
if strings.Contains(strings.ToLower(query.SQL), "select 1") {
p.pgspecial = pgSelect1
p.Name = "SELECT"
} else {
switch query.SQLType {
case sql.SQLTypeSet:
p.Name = "SET"
set := query.Statement.(*sqlparser.Set)
p.pgspecial = 0
for _, item := range set.Exprs {
if item.Name.String() == "application_name" {
switch item.Expr.(type) {
case *sqlparser.SQLVal:
p.pgspecial = pgSetApplication
}
}
}
case sql.SQLTypeSelect:
p.Name = "SELECT"
p.pgspecial = pgPassOn
stmt := query.Statement.(*sqlparser.Select)
for _, item := range stmt.SelectExprs {
switch expr := item.(type) {
case *sqlparser.AliasedExpr:
switch colExpr := expr.Expr.(type) {
case *sqlparser.FuncExpr:
funcName := strings.ToLower(colExpr.Name.String())
switch funcName {
case "pg_backend_pid":
//SELECT pg_backend_pid()
p.pgspecial = pgBackendPid
case "pg_terminate_backend":
//select pg_terminate_backend(100)
p.pgspecial = pgTerminate
case "version":
//SELECT VERSION() AS version
p.pgspecial = pgVersion
}
//need to return the pid from the cancelation object
//add row description object
//add data row for item
}
}
}
for _, item := range stmt.From {
switch from := item.(type) {
case *sqlparser.AliasedTableExpr:
tableName := from.Expr.(sqlparser.TableName).ToViewName().Name.String()
switch tableName {
case "pg_type":
p.pgspecial = pgCountType
case "pg_stat_activity":
p.pgspecial = pgQueryTime
case "tables":
p.pgspecial = pgSchema
}
}
}
p.sql = queryStr
case sql.SQLTypeBegin:
// Ignore BEGIN
p.pgspecial = pgBegin
case sql.SQLTypeShow:
p.Name = "SHOW"
p.pgspecial = pgPassOn
p.sql = queryStr
}
}
} else {
p.pgspecial = pgEmpty
}
p.Add(message.ParseOK)
}
func (p *Portal) Describe() {
// Placeholder should we need to handle the Decribe request
}
func (p *Portal) Execute() (shouldTerminate bool, queryReady bool, err error) {
queryReady = true
switch p.pgspecial {
case pgBackendPid:
rowDescription, e := p.Encoder.EncodeColumn("pg_backend_pid", int32(23), 4)
if e != nil {
err = e
return
}
p.Add(rowDescription)
pid := fmt.Sprintf("%v", p.pid)
dataRow, _ := p.Encoder.TextRow(pid)
p.Add(dataRow)
//needs data row with cancel token
case pgVersion:
rowDescription, e := p.Encoder.EncodeColumn("version", int32(25), -1)
if e != nil {
err = e
return
}
p.Add(rowDescription)
mesg := fmt.Sprintf("PostgresSQL 13.0 (molecula.%v)", p.server.QueryHandler.Version())
dataRow, _ := p.Encoder.TextRow(mesg)
p.Add(dataRow)
case pgSelect1:
rowDescription, e := p.Encoder.EncodeColumn("?column?", int32(23), 4)
if e != nil {
err = e
return
}
p.Add(rowDescription)
dataRow, _ := p.Encoder.TextRow("1")
p.Add(dataRow)
case pgCountType:
//need to block
<-p.server.lookerChannel
rowDescription, e := p.Encoder.EncodeColumn("count", int32(20), 8)
if err != nil {
err = e
return
}
p.Add(rowDescription)
errorResponse, _ := p.Encoder.Error(
message.NoticeField{
Type: message.NoticeFieldSeverity,
Data: "FATAL",
},
message.NoticeField{
Type: message.NoticeFieldMessage,
Data: "terminating connection due to administrator command",
},
message.NoticeField{
Type: message.NoticeFieldCode,
Data: "57P01",
},
)
p.Add(errorResponse)
e = p.Sync() //send and Reset
if e != nil {
err = e
return
}
return true, queryReady, nil
case pgQueryTime:
// need to return something so that the id can be queried
//need to return SELECT pid as id, query as stmt, EXTRACT(seconds from query_start - NOW()) as elapsed_time FROM pg_stat_activity
//seems like we need a map of pids to querys
e := p.server.dumpPortalsTo(p)
if e != nil {
err = e
return
}
case pgTerminate:
//note just have 1 lock that blocks all who try to count the activities
//TODO (twg) lock this
close(p.server.lookerChannel) //release all the other blockers and allow them to terminate
p.server.lookerChannel = make(chan struct{}) //create a new one just in case
// i think it needs to return boolean true
rowDescription, e := p.Encoder.EncodeColumn("pg_terminate_backend", int32(16), 1)
if e != nil {
err = e
return
}
p.Add(rowDescription)
dataRow, _ := p.Encoder.TextRow("t")
p.Add(dataRow)
commandComplete, e := p.Encoder.CommandComplete("SELECT 1")
if e != nil {
err = e
return
}
p.Add(commandComplete)
e = p.Sync()
if e != nil {
err = e
return
}
return false, queryReady, nil
case pgEmpty:
p.Add(message.NoData)
p.Add(message.EmptyQueryResponse)
e := p.Sync()
if e != nil {
err = e
return
}
queryReady = true
return false, queryReady, nil
case pgSetApplication:
//needs to add/send status
msg, e := p.Encoder.ParameterStatus("application_name", "PostgreSQL JDBC Driver")
if e != nil {
err = e
return
}
p.Add(msg)
case pgSchema:
parts := []message.SimpleColumn{
{
Name: "table_schema",
Typeid: int32(19),
Typelen: 64,
},
{
Name: "table_name",
Typeid: int32(19),
Typelen: 64,
},
}
rowDescription, e := p.Encoder.EncodeColumns(parts...)
if e != nil {
err = e
return
}
p.Add(rowDescription)
e = p.HandleSchema()
if e != nil {
err = e
return
}
case pgPassOn:
query := SimpleQuery(p.sql)
e := p.server.handleQuery(p, query, p.cancelNotify)
if e != nil {
err = e
return
}
return
case pgBegin:
p.Name = "BEGIN"
}
//maybe add in the number of items in select clause
if len(p.Name) > 0 { //only send command complete for those that have names
message, _ := p.Encoder.CommandComplete(p.Name)
p.Add(message)
}
return
}
// handleStandard handles a connection in the standard postgres wire protocol.
func (p *Portal) Sync() error {
for _, m := range p.commands {
err := p.Writer.WriteMessage(m)
if err != nil {
return err
}
}
p.Writer.Flush()
p.Reset()
return nil
}
func (p *Portal) Add(m message.Message) {
cp := message.Message{Type: m.Type, Data: make([]byte, len(m.Data))}
copy(cp.Data, m.Data)
p.commands = append(p.commands, cp)
}
func (p *Portal) DumpComands() {
for _, m := range p.commands {
m.Dump("DUMPING:")
}
}
func (p *Portal) HandleSchema() error {
return p.server.QueryHandler.HandleSchema(context.Background(), p)
}
func (p *Portal) WriteMessage(m message.Message) error {
p.Add(m)
return nil
}
func (p *Portal) Flush() error {
return nil
}
// handleStandard handles a connection in the standard postgres wire protocol.
// The client is responsible for closing the connection when this finishes.
@ -255,7 +632,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
if err != nil {
return errors.Wrap(err, "parsing parameters")
}
if user, ok := params["user"]; ok {
// Log the connection.
s.Logger.Debugf("new postgres connection from user %q at %v", user, conn.RemoteAddr())
@ -265,7 +641,6 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
}
// Set up message input and output.
// Set up a reader that will preempt the connection when the context is canceled.
ir := idleReader{
conn: conn,
@ -322,8 +697,56 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
if err != nil {
return errors.Wrap(err, "sending authentication confirmation")
}
err = s.SendParameterStatus(w, "application_name", "", &encoder)
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
err = s.SendParameterStatus(w, "client_encoding", "UTF8", &encoder)
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
err = s.SendParameterStatus(w, "DateStyle", "ISO, MDY", &encoder)
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
err = s.SendParameterStatus(w, "integer_datetimes", "on", &encoder)
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
err = s.SendParameterStatus(w, "IntervalStyle", "postgres", &encoder)
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
err = s.SendParameterStatus(w, "is_superuser", "on", &encoder)
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
err = s.SendParameterStatus(w, "server_encoding", "UTF8", &encoder)
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
err = s.SendParameterStatus(w, "server_version", PgServerVersion, &encoder)
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
/*
-PARAMETER STATUS name='TimeZone', value='GMT'
*/
err = s.SendParameterStatus(w, "session_authorization", "docker", &encoder) //TODO(twg) figure out valid values here
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
err = s.SendParameterStatus(w, "standard_conforming_strings", "on", &encoder) //TODO(twg) figure out valid values here
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
err = s.SendParameterStatus(w, "TimeZone", "GMT", &encoder) //TODO(twg) figure out valid values here
if err != nil {
return errors.Wrap(err, "sending parameter status server version")
}
var cancelNotify <-chan struct{}
var pid int32
if s.CancellationManager != nil {
notify, cancel, token, err := s.CancellationManager.Token()
if err != nil {
@ -332,6 +755,7 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
defer cancel()
msg, err := encoder.BackendKeyData(token.PID, token.Key)
pid = token.PID
if err != nil {
return errors.Wrap(err, "encoding cancellation key data")
}
@ -343,11 +767,23 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
}
var queryReady bool
portal := &Portal{
Writer: w,
Encoder: &encoder,
commands: make([]message.Message, 0),
mapper: sql.NewMapper(),
pid: pid,
server: s,
cancelNotify: cancelNotify,
}
s.addPortal(portal)
defer s.removePortal(portal)
//mapper.Logger = logger
for {
if !queryReady {
// Indicate that we are ready for a query.
// TODO: provide a valid transaction state.
msg, err := encoder.ReadyForQuery(message.TransactionStatusActive)
portal.sql = ""
msg, err := encoder.ReadyForQuery(message.TransactionStatusIdle)
if err != nil {
return errors.Wrap(err, "sending query ready status")
}
@ -373,6 +809,7 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
// However, it seems that no clients completely follow the spec, so we shouldn't rely on anything that isn't entirely straightforward.
s.Logger.Debugf("postgres client sent additional data without waiting for completion")
}
queryReady = true
}
// Read the next packet.
@ -406,12 +843,27 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
// We are done.
return w.Flush()
case message.TypeSimpleQuery:
// Execute a simple query.
case message.TypeParse:
portal.Parse(msg.Data)
case message.TypeBind:
portal.Bind()
case message.TypeExecute:
term, qr, err := portal.Execute()
if err != nil {
return err
}
if term {
return w.Flush()
}
queryReady = qr
case message.TypeSync:
err := portal.Sync()
if err != nil {
return err
}
queryReady = false
case message.TypeSimpleQuery:
queryReady = false
// Parse the query message (a null-terminated string).
query := SimpleQuery(strings.TrimSuffix(string(msg.Data), "\x00"))
// Execute the query.
@ -419,7 +871,10 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
if err != nil {
return err
}
case message.TypeDescribe:
portal.Describe()
case message.TypeClose:
return w.Flush()
default:
// The message is not supported yet.
// Send an error.
@ -452,6 +907,70 @@ func (s *Server) handleStandard(ctx context.Context, proto Protocol, conn net.Co
}
}
}
func (s *Server) addPortal(p *Portal) {
s.mu.Lock()
defer s.mu.Unlock()
s.portals = append(s.portals, p)
}
func (s *Server) removePortal(p *Portal) {
s.mu.Lock()
defer s.mu.Unlock()
for i, portal := range s.portals {
if portal.pid == p.pid {
//remove i
s.portals = append(s.portals[:i], s.portals[i+1:]...)
return
}
}
}
func (s *Server) dumpPortalsTo(p *Portal) error {
s.mu.Lock()
defer s.mu.Unlock()
//need to add the descrition for the 3 fields
// <-:-ROW DESCRIPTION: num_fields=3
//---[Field 01]--- name='id' type=23 type_len=4 type_mod=4294967295 relid=12250 attnum=3 format=0
//---[Field 02]--- name='stmt' type=25 type_len=65535 type_mod=4294967295 relid=12250 attnum=20 format=0 -
//--[Field 03]--- name='elapsed_time' type=701 type_len=8 type_mod=4294967295 relid=0 attnum=0 format=0
parts := []message.SimpleColumn{
{
Name: "id",
Typeid: int32(23),
Typelen: 4,
},
{
Name: "stmt",
Typeid: int32(25),
Typelen: -1,
},
{
Name: "elapsed_time",
Typeid: int32(701),
Typelen: 8,
},
}
rowDescription, err := p.Encoder.EncodeColumns(parts...)
if err != nil {
return err
}
p.Add(rowDescription)
for _, portal := range s.portals {
dataRow, err := p.Encoder.TextRow(
fmt.Sprintf("%v", portal.pid),
portal.sql,
fmt.Sprintf("%v", time.Since(portal.queryStart).Seconds()))
if err != nil {
return err
}
//if sql == "" need to put in a null record
//need to add the dararow
//also need to figure out null types
p.Add(dataRow)
}
return nil
}
// handleQuery processes a single query on a connection.
func (s *Server) handleQuery(w message.Writer, query Query, cancelNotify <-chan struct{}) error {
@ -497,7 +1016,6 @@ func (s *Server) handleQuery(w message.Writer, query Query, cancelNotify <-chan
te: s.TypeEngine,
tag: "SELECT",
}
// Dispatch the query handler.
qerr := s.QueryHandler.HandleQuery(ctx, qwriter, query)
if qerr != nil {

2
pg/query.go
View file

@ -60,6 +60,8 @@ type QueryResultWriter interface {
type QueryHandler interface {
// HandleQuery executes a query and writes the results back.
HandleQuery(context.Context, QueryResultWriter, Query) error
HandleSchema(context.Context, *Portal) error
Version() string
}
// queryResultWriter implements QueryResultWrtiter over postgres wire protocol.

5
pg/server.go
View file

@ -59,6 +59,9 @@ type Server struct {
// CancellationManager is the cancellation manager to use.
// If this is not set, no cancellations will be applied.
CancellationManager CancellationManager
lookerChannel chan struct{}
mu sync.Mutex
portals []*Portal
}
// ServeConn serves a single connection.
@ -79,6 +82,8 @@ func (s *Server) Serve(ctx context.Context, l net.Listener) (err error) {
err = cerr
}
}(ctx)
// TODO (twg) added for looker
s.lookerChannel = make(chan struct{})
// Wait for the listener to be closed and all connections to shut down.
var wg sync.WaitGroup

917
planner.go
View file

File diff suppressed because it is too large Load diff

298
planner_test.go
View file

@ -210,60 +210,308 @@ func TestPlanner_Select(t *testing.T) {
}
t.Run("UnqualifiedColumns", func(t *testing.T) {
results := mustQueryRows(t, c.GetNode(0).Server, `SELECT _id, a, b FROM i0`)
if diff := cmp.Diff(results, [][]interface{}{
{int64(1), int64(10), int64(100)},
{int64(2), int64(20), int64(200)},
}); diff != "" {
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT a, b, _id FROM i0`)
if diff := cmp.Diff([][]interface{}{
{int64(10), int64(100), int64(1)},
{int64(20), int64(200), int64(2)},
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "a", Type: "INT"},
{Name: "b", Type: "INT"},
{Name: "_id", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("QualifiedColumns", func(t *testing.T) {
results := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0._id, i0.a, i0.b FROM i0`)
if diff := cmp.Diff(results, [][]interface{}{
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0._id, i0.a, i0.b FROM i0`)
if diff := cmp.Diff([][]interface{}{
{int64(1), int64(10), int64(100)},
{int64(2), int64(20), int64(200)},
}); diff != "" {
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "_id", Type: "INT"},
{Name: "a", Type: "INT"},
{Name: "b", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("UnqualifiedStar", func(t *testing.T) {
results := mustQueryRows(t, c.GetNode(0).Server, `SELECT * FROM i0`)
if diff := cmp.Diff(results, [][]interface{}{
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT * FROM i0`)
if diff := cmp.Diff([][]interface{}{
{int64(1), int64(10), int64(100)},
{int64(2), int64(20), int64(200)},
}); diff != "" {
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "_id", Type: "INT"},
{Name: "a", Type: "INT"},
{Name: "b", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("QualifiedStar", func(t *testing.T) {
results := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0.* FROM i0`)
if diff := cmp.Diff(results, [][]interface{}{
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT i0.* FROM i0`)
if diff := cmp.Diff([][]interface{}{
{int64(1), int64(10), int64(100)},
{int64(2), int64(20), int64(200)},
}); diff != "" {
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "_id", Type: "INT"},
{Name: "a", Type: "INT"},
{Name: "b", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("NoIdentifier", func(t *testing.T) {
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT a, b FROM i0`)
if diff := cmp.Diff([][]interface{}{
{int64(10), int64(100)},
{int64(20), int64(200)},
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "a", Type: "INT"},
{Name: "b", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("ErrFieldNotFound", func(t *testing.T) {
stmt, err := c.GetNode(0).Server.PlanSQL(context.Background(), `SELECT xyz FROM i0`)
if err != nil {
t.Fatal(err)
}
defer stmt.Close()
var xyz interface{}
if err := stmt.QueryRowContext(context.Background()).Scan(&xyz); err == nil || !strings.Contains(err.Error(), `xyz: field not found`) {
_, err := c.GetNode(0).Server.PlanSQL(context.Background(), `SELECT xyz FROM i0`)
if err == nil || !strings.Contains(err.Error(), `xyz: field not found`) {
t.Fatalf("unexpected error: %v", err)
}
})
}
func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{} {
func TestPlanner_GroupBy(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
defer i0.Close()
if _, err := i0.CreateField("x"); err != nil {
t.Fatal(err)
} else if _, err := i0.CreateField("y", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
t.Fatal(err)
} else if _, err := i0.CreateField("z", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
t.Fatal(err)
}
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Query: `
Set(1, x=10)
Set(1, x=20)
Set(1, y=100)
Set(1, z=500)
Set(2, x=10)
Set(2, y=200)
Set(2, z=500)
Set(3, x=20)
Set(3, z=600)
`}); err != nil {
t.Fatal(err)
}
t.Run("Count", func(t *testing.T) {
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*), x FROM i0 GROUP BY x`)
if diff := cmp.Diff([][]interface{}{
{int64(2), int64(10)},
{int64(2), int64(20)},
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "count", Type: "INT"},
{Name: "x", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("DistinctCount", func(t *testing.T) {
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(DISTINCT z), x FROM i0 GROUP BY x`)
if diff := cmp.Diff([][]interface{}{
{int64(1), int64(10)},
{int64(2), int64(20)},
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "count", Type: "INT"},
{Name: "x", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("Sum", func(t *testing.T) {
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT sum(y), x FROM i0 GROUP BY x`)
if diff := cmp.Diff([][]interface{}{
{int64(300), int64(10)},
{int64(100), int64(20)},
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "sum", Type: "INT"},
{Name: "x", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("ReorderColumns", func(t *testing.T) {
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT x, COUNT(*) FROM i0 GROUP BY x`)
if diff := cmp.Diff([][]interface{}{
{int64(10), int64(2)},
{int64(20), int64(2)},
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "x", Type: "INT"},
{Name: "count", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("NoResultColumn", func(t *testing.T) {
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 GROUP BY x`)
if diff := cmp.Diff([][]interface{}{
{int64(2)},
{int64(2)},
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "count", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
}
func TestPlanner_InnerJoin(t *testing.T) {
c := test.MustRunCluster(t, 1)
defer c.Close()
i0, err := c.GetHolder(0).CreateIndex("i0", pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
defer i0.Close()
if _, err := i0.CreateField("a", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
t.Fatal(err)
}
i1, err := c.GetHolder(0).CreateIndex("i1", pilosa.IndexOptions{TrackExistence: true})
if err != nil {
t.Fatal(err)
}
defer i1.Close()
if _, err := i1.CreateField("parentid", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
t.Fatal(err)
} else if _, err := i1.CreateField("x", pilosa.OptFieldTypeInt(0, 1000)); err != nil {
t.Fatal(err)
}
// Populate with data.
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i0",
Query: `
Set(1, a=10)
Set(2, a=20)
Set(3, a=30)
`}); err != nil {
t.Fatal(err)
}
if _, err := c.GetNode(0).API.Query(context.Background(), &pilosa.QueryRequest{
Index: "i1",
Query: `
Set(1, parentid=1)
Set(1, x=100)
Set(2, parentid=1)
Set(2, x=200)
Set(3, parentid=2)
Set(3, x=300)
`}); err != nil {
t.Fatal(err)
}
t.Run("Count", func(t *testing.T) {
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid`)
if diff := cmp.Diff([][]interface{}{
{int64(2)},
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "count", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
t.Run("CountWithParentCondition", func(t *testing.T) {
results, columns := mustQueryRows(t, c.GetNode(0).Server, `SELECT COUNT(*) FROM i0 INNER JOIN i1 ON i0._id = i1.parentid WHERE i0.a = 10`)
if diff := cmp.Diff([][]interface{}{
{int64(1)},
}, results); diff != "" {
t.Fatal(diff)
}
if diff := cmp.Diff([]*pilosa.StmtColumn{
{Name: "count", Type: "INT"},
}, columns); diff != "" {
t.Fatal(diff)
}
})
}
func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) (results [][]interface{}, columns []*pilosa.StmtColumn) {
tb.Helper()
stmt, err := svr.PlanSQL(context.Background(), q)
@ -277,7 +525,7 @@ func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{}
tb.Fatal(err)
}
results := make([][]interface{}, 0)
results = make([][]interface{}, 0)
for rows.Next() {
result := make([]interface{}, len(rows.Columns()))
@ -297,5 +545,5 @@ func mustQueryRows(tb testing.TB, svr *pilosa.Server, q string) [][]interface{}
tb.Fatal(err)
}
return results
return results, rows.Columns()
}

2
rbf/db.go
View file

@ -274,7 +274,7 @@ func (db *DB) Close() (err error) {
// Close writer handler.
if db.file != nil {
err = db.file.Sync()
err = db.fsync(db.file)
if err != nil {
return
}

61
roaring/roaring.go
View file

@ -2997,6 +2997,7 @@ func ArrayCountRange(array []uint16, start, end int32) (n int32) {
return n
}
// BitmapCountRange counts bits set in [start,end).
func BitmapCountRange(bitmap []uint64, start, end int32) int32 {
if roaringParanoia {
if start > end {
@ -3033,15 +3034,16 @@ func BitmapCountRange(bitmap []uint64, start, end int32) int32 {
}
func callbackBits(w uint64, base uint16, fn func(uint16)) {
bit := uint16(0)
for w != 0 {
trail := bits.TrailingZeros64(w)
bit += uint16(trail)
w >>= (trail + 1)
fn(base + bit)
trail := uint16(bits.TrailingZeros64(w))
fn(base + trail)
base += trail + 1
w >>= trail + 1
}
}
// bitmapCallbackRange calls the provided function for every bit set in
// bitmap in the range [start,end).
func bitmapCallbackRange(bitmap []uint64, start, end int32, fn func(uint16)) {
if roaringParanoia {
if start > end {
@ -3051,11 +3053,30 @@ func bitmapCallbackRange(bitmap []uint64, start, end int32, fn func(uint16)) {
i, j := start/64, end/64
// Special case when start and end fall in the same word.
if i == j {
offi, offj := uint(start%64), uint(64-end%64)
w := (bitmap[i] >> offi) << (offj + offi)
// So, we want to know the offsets. For instance, if start and end
// are 65 and 69, we might want i=1, offi=1, j=1, offj=5. Then we
// compute masks from offi (masking out 0x1, or (1<<offi)-1), and
// offj (masking out everything *but* (1<<offj)-1).
//
// But we can also use "x >> offi << offi" to trim the lowest offi
// bits, and "x << (64-offj) >> (64-offj)" to trim all but the
// lowest offj bits.
//
// We can then simplify slightly further: we use the inverted value
// as offj, and compute (w << offi) >> (offi + offj) << offi.
//
// But wait, you ask. What if offi+offj is too large! Well, then
// start and end were in the wrong order. We have 0 <= i <= j < 64.
// If x+i > 64, then x > (64-i). Thus, if (64-j)+i > 64, it
// follows that (64-j) > (64-i). So they'd have been in the wrong order.
// In which case, we correctly yield a value of (0 << offi), or 0,
// because nothing is between them.
offi, offj := uint(start%64), uint(64-(end%64))
w := (bitmap[i] << offj) >> (offi + offj) << offi
if w != 0 {
callbackBits(w, uint16(i)*64, fn)
}
return
}
// Count partial starting word.
@ -4492,7 +4513,7 @@ func intersectionCallbackArrayArray(a, b *Container, fn func(uint16)) {
}
if (na << 2) < nb {
for _, va := range ca {
for cb[0] < va {
if cb[0] < va {
// try to skip ahead a bit faster
for len(cb) > 7 && cb[7] < va {
cb = cb[8:]
@ -4530,19 +4551,21 @@ func intersectionCallbackArrayRun(a, b *Container, fn func(uint16)) {
na, nb := len(array), len(runs)
for i, j := 0, 0; i < na && j < nb; {
va, vb := array[i], runs[j]
if va < vb.Start {
i++
} else if va >= vb.Start && va <= vb.Last {
i++
fn(va)
} else if va > vb.Last {
if va > vb.Last {
j++
continue
}
// If we got here, va is either before or in the current run,
// so we're definitely done with this member of the array.
i++
if va >= vb.Start {
fn(va)
}
}
}
func intersectionCallbackRunRun(a, b *Container, fn func(uint16)) {
statsHit("intersectionCount/RunRun")
statsHit("intersectionCallback/RunRun")
ra, rb := a.runs(), b.runs()
na, nb := len(ra), len(rb)
for i, j := 0, 0; i < na && j < nb; {
@ -4582,14 +4605,14 @@ func intersectionCallbackRunRun(a, b *Container, fn func(uint16)) {
}
func intersectionCallbackBitmapRun(a, b *Container, fn func(uint16)) {
statsHit("intersectionCount/BitmapRun")
statsHit("intersectionCallback/BitmapRun")
for _, iv := range b.runs() {
bitmapCallbackRange(a.bitmap(), int32(iv.Start), int32(iv.Last)+1, fn)
}
}
func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) {
statsHit("intersectionCount/ArrayBitmap")
statsHit("intersectionCallback/ArrayBitmap")
bitmap := b.bitmap()
ln := len(bitmap)
for _, val := range a.array() {
@ -4598,14 +4621,14 @@ func intersectionCallbackArrayBitmap(a, b *Container, fn func(uint16)) {
break
}
off := val % 64
if (bitmap[i]>>off) & 1 != 0 {
if (bitmap[i]>>off)&1 != 0 {
fn(val)
}
}
}
func intersectionCallbackBitmapBitmap(a, b *Container, fn func(uint16)) {
statsHit("intersectionCount/BitmapBitmap")
statsHit("intersectionCallback/BitmapBitmap")
ab, bb := a.bitmap(), b.bitmap()
for i := range ab {
w := ab[i] & bb[i]

100
roaring/roaring_internal_test.go
View file

@ -4636,3 +4636,103 @@ func TestContainer_unionInPlace_ArrayUnionRun(t *testing.T) {
}
}
}
func TestContainerCallback(t *testing.T) {
containers, err := InitContainerArchetypes()
if err != nil {
t.Fatalf("creating containers: %v", err)
}
got := make([]uint16, 65536)
hit := func(u uint16) {
got = append(got, u)
}
var expected []uint16
// complain() wraps up some pretty-printing logic for this,
// but note also the closure trapping expected/got so we can
// just refer to them without passing them in.
complain := func(t *testing.T, msg string, args ...interface{}) {
l1 := len(expected)
l2 := len(got)
dotdot1 := ""
dotdot2 := ""
if l1 > 8 {
expected = expected[:8]
dotdot1 = "..."
}
if l2 > 8 {
got = got[:8]
dotdot2 = "..."
}
t.Fatalf("%s: expected %d%s, got %d%s", fmt.Sprintf(msg, args...), expected, dotdot1, got, dotdot2)
}
for t1, ci := range containers {
t.Run(ContainerArchetypeNames[t1], func(t *testing.T) {
for _, c1 := range ci {
got = got[:0]
expected = c1.Slice()
containerCallback(c1, hit)
if len(got) != len(expected) {
complain(t, "wrong length (%d vs %d)", len(expected), len(got))
}
for i := range got {
if got[i] != expected[i] {
complain(t, "element %d differs: expected %d, got %d", i, expected[i], got[i])
}
}
}
})
}
}
func TestIntersectionCallback(t *testing.T) {
containers, err := InitContainerArchetypes()
if err != nil {
t.Fatalf("creating containers: %v", err)
}
got := make([]uint16, 65536)
hit := func(u uint16) {
got = append(got, u)
}
var expected []uint16
// complain() wraps up some pretty-printing logic for this,
// but note also the closure trapping expected/got so we can
// just refer to them without passing them in.
complain := func(t *testing.T, msg string, args ...interface{}) {
l1 := len(expected)
l2 := len(got)
dotdot1 := ""
dotdot2 := ""
if l1 > 8 {
expected = expected[:8]
dotdot1 = "..."
}
if l2 > 8 {
got = got[:8]
dotdot2 = "..."
}
t.Fatalf("%s: expected %d%s, got %d%s", fmt.Sprintf(msg, args...), expected, dotdot1, got, dotdot2)
}
for t1, ci := range containers {
for t2, cj := range containers {
t.Run(fmt.Sprintf("%s-%s", ContainerArchetypeNames[t1], ContainerArchetypeNames[t2]), func(t *testing.T) {
for _, c1 := range ci {
for _, c2 := range cj {
got = got[:0]
expectedContainer := intersect(c1, c2)
expected = expectedContainer.Slice()
intersectionCallback(c1, c2, hit)
if len(got) != len(expected) {
complain(t, "wrong length (%d vs %d)", len(expected), len(got))
}
for i := range got {
if got[i] != expected[i] {
complain(t, "element %d differs: expected %d, got %d", i, expected[i], got[i])
}
}
}
}
})
}
}
}

16
server.go
View file

@ -339,6 +339,9 @@ func OptServerOpenTranslateReader(fn OpenTranslateReaderFunc) ServerOption {
func OptServerStorageConfig(cfg *storage.Config) ServerOption {
return func(s *Server) error {
s.holderConfig.StorageConfig = cfg
// For historical reasons, RBF's config can ignore the storage config
// in some cases.
s.holderConfig.RBFConfig.FsyncEnabled = s.holderConfig.StorageConfig.FsyncEnabled
return nil
}
}
@ -437,7 +440,7 @@ func NewServer(opts ...ServerOption) (*Server, error) {
confirmDownRetries: defaultConfirmDownRetries,
confirmDownSleep: defaultConfirmDownSleep,
resetTranslationSyncCh: make(chan struct{}),
resetTranslationSyncCh: make(chan struct{}, 1),
logger: logger.NopLogger,
}
@ -569,11 +572,6 @@ func (s *Server) Open() error {
log.Println(errors.Wrap(err, "logging startup"))
}
// Start background process listening for translation
// sync resets.
s.wg.Add(1)
go func() { defer s.wg.Done(); s.monitorResetTranslationSync() }()
// Start DisCo.
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
@ -612,6 +610,12 @@ func (s *Server) Open() error {
s.syncer.Closing = s.closing
s.syncer.Stats = s.holder.Stats.WithTags("component:HolderSyncer")
// Start background process listening for translation
// sync resets.
s.wg.Add(1)
go func() { defer s.wg.Done(); s.monitorResetTranslationSync() }()
go func() { _ = s.translationSyncer.Reset() }()
// Open holder.
func() {
s.holder.startMsgsMu.Lock()

3
server/config.go
View file

@ -200,6 +200,9 @@ type Config struct {
// Setting this to 0 disables the limit.
// This mostly exists because other DBs seem to have it.
ConnectionLimit uint16 `toml:"max-connections"`
// SqlVersion is which type of sqlhandling to be applied.
// The constant SqlV2 can be used to try the new experimental Molecula SQL handling
SqlVersion uint16 `toml:"sql-version"`
} `toml:"postgres"`
// Storage.Backend determines which Tx implementation the holder/Index will

172
server/pg.go
View file

@ -25,9 +25,11 @@ import (
"strings"
"time"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/pg"
//"github.com/molecula/featurebase/v2/pg"
"github.com/molecula/featurebase/v2/pql"
pb "github.com/molecula/featurebase/v2/proto"
@ -43,14 +45,20 @@ type PostgresServer struct {
s pg.Server
stop context.CancelFunc
}
type SqlVersion uint16
const (
SqlV1 SqlVersion = 0
SqlV2 SqlVersion = 2
)
// NewPostgresServer creates a postgres server.
func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config) *PostgresServer {
func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config, sqlVersion SqlVersion) *PostgresServer {
return &PostgresServer{
api: api,
logger: logger,
s: pg.Server{
QueryHandler: NewPostgresHandler(api, logger),
QueryHandler: NewPostgresHandler(api, logger, sqlVersion),
TypeEngine: pg.PrimitiveTypeEngine{},
StartupTimeout: 5 * time.Second,
ReadTimeout: 10 * time.Second,
@ -66,11 +74,12 @@ func NewPostgresServer(api *pilosa.API, logger logger.Logger, tls *tls.Config) *
}
// NewPostgresHandler creates a postgres query handler wrapping the pilosa API.
func NewPostgresHandler(api *pilosa.API, logger logger.Logger) pg.QueryHandler {
return &queryDecodeHandler{
child: &pilosaQueryHandler{
api: api,
logger: logger,
func NewPostgresHandler(api *pilosa.API, logger logger.Logger, sqlVersion SqlVersion) pg.QueryHandler {
return &QueryDecodeHandler{
Child: &PilosaQueryHandler{
Api: api,
logger: logger,
sqlVersion: sqlVersion,
},
}
}
@ -107,6 +116,10 @@ func (s *PostgresServer) Close() error {
return s.eg.Wait()
}
func (s *PostgresServer) GetAPI() *pilosa.API {
return s.api
}
type pgPQLQuery struct {
index string
query string
@ -135,9 +148,10 @@ func pgDecodePQL(str string) (q pg.Query, err error) {
}, nil
}
type pilosaQueryHandler struct {
api *pilosa.API
logger logger.Logger
type PilosaQueryHandler struct {
Api *pilosa.API
logger logger.Logger
sqlVersion SqlVersion
}
func pgWriteRow(w pg.QueryResultWriter, row *pilosa.Row) error {
@ -322,6 +336,78 @@ func pgWriteGroupCount(w pg.QueryResultWriter, counts *pilosa.GroupCounts) error
return nil
}
func pgWriteStmtRows(w pg.QueryResultWriter, rows *pilosa.StmtRows) error {
//TODO(twg) writeHeader
//TODO(twg) writeColumns
first := true
var data []string
var err error
for rows.Next() {
if first {
columns := rows.Columns()
//TODO (twg) types:=rows.Types()
headers := make([]pg.ColumnInfo, len(columns))
for i, column := range columns {
headers[i] = pg.ColumnInfo{
Name: column.Name,
Type: pg.TypeCharoid, //TODO(twg) types[i]
}
}
err := w.WriteHeader(headers...)
if err != nil {
return err
}
data = make([]string, len(headers))
first = false
}
result := make([]interface{}, len(rows.Columns()))
// Create list of scan destination pointers.
dsts := make([]interface{}, len(result))
for i := range result {
dsts[i] = &result[i]
}
if err := rows.Scan(dsts...); err != nil {
return err
}
//TODO(twg) conversion should be happening in Scan as described in https://pkg.go.dev/database/sql
// ....
// Scan also converts between string and numeric types, as long as no information
// would be lost. While Scan stringifies all numbers scanned from numeric database columns into *string,
// scans into numeric types are checked for overflow. For example, a float64 with value 300 or a string
// with value "300" can scan into a uint16, but not into a uint8, though float64(255) or "255" can scan
// into a uint8. One exception is that scans of some float64 numbers to strings may lose information when stringifying.
// In general, scan floating point columns into *float64.
// ...
for i, col := range dsts {
var v string
switch col := col.(type) {
case nil:
v = "null"
//
//v = strconv.FormatUint(col.Uint64Val, 10)
case *interface{}:
v = fmt.Sprintf("%v", *col)
default:
return errors.Errorf("unable to process value of type %T", col)
}
data[i] = v
}
err = w.WriteRowText(data...)
if err != nil {
return err
}
}
if err := rows.Err(); err != nil {
return err
}
return nil
}
func pgWriteRowser(w pg.QueryResultWriter, result pb.ToRowser) error {
var data []string
return result.ToRows(func(row *pb.RowResponse) error {
@ -395,6 +481,8 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error {
return pgWriteGroupCount(w, result)
case pb.ToRowser: // we should avoid protobuf where we can...
return pgWriteRowser(w, result)
case *pilosa.StmtRows: // we should avoid protobuf where we can...
return pgWriteStmtRows(w, result)
case uint64:
err := w.WriteHeader(pg.ColumnInfo{
Name: "count",
@ -449,10 +537,10 @@ func pgWriteResult(w pg.QueryResultWriter, result interface{}) error {
}
}
func (pqh *pilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
func (pqh *PilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
switch q := q.(type) {
case pgPQLQuery:
resp, err := pqh.api.Query(ctx, &pilosa.QueryRequest{
resp, err := pqh.Api.Query(ctx, &pilosa.QueryRequest{
Index: q.index,
Query: q.query,
})
@ -465,22 +553,36 @@ func (pqh *pilosaQueryHandler) HandleQuery(ctx context.Context, w pg.QueryResult
return errors.Wrap(pgWriteResult(w, resp.Results[0]), "writing query result")
case pg.SimpleQuery:
resp, err := execSQL(ctx, pqh.api, pqh.logger, string(q))
if err != nil {
return errors.Wrap(err, "executing query")
if pqh.sqlVersion == SqlV2 {
stmt, err := pqh.Api.Plan(ctx, string(q))
if err != nil {
return err
}
resp, err := stmt.QueryContext(ctx)
if err != nil {
return err
}
return errors.Wrap(pgWriteResult(w, resp), "writing sql2 query result")
//version 2.0
} else {
//version 1.0
resp, err := execSQL(ctx, pqh.Api, pqh.logger, string(q))
if err != nil {
return errors.Wrap(err, "executing query")
}
return errors.Wrap(pgWriteResult(w, resp), "writing query result")
}
return errors.Wrap(pgWriteResult(w, resp), "writing query result")
default:
return errors.Errorf("query type %T not yet supported (query: %s)", q, q)
}
}
type queryDecodeHandler struct {
child pg.QueryHandler
type QueryDecodeHandler struct {
Child pg.QueryHandler
}
func (qdh *queryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
func (qdh *QueryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResultWriter, q pg.Query) error {
switch qv := q.(type) {
case pg.SimpleQuery:
if strings.HasPrefix(string(qv), "[") {
@ -492,5 +594,33 @@ func (qdh *queryDecodeHandler) HandleQuery(ctx context.Context, w pg.QueryResult
}
}
return qdh.child.HandleQuery(ctx, w, q)
return qdh.Child.HandleQuery(ctx, w, q)
}
func (qdh *QueryDecodeHandler) Version() string {
return qdh.Child.Version()
}
func (pqh *PilosaQueryHandler) Version() string {
if pqh.sqlVersion > 0 {
return "v2"
}
return "v1"
}
func (pqh *PilosaQueryHandler) HandleSchema(ctx context.Context, portal *pg.Portal) error {
schema, err := pqh.Api.Schema(context.Background(), false)
if err != nil {
return err
}
for _, ii := range schema {
dataRow, err := portal.Encoder.TextRow("featurebase", ii.Name)
if err != nil {
return err
}
portal.Add(dataRow)
}
return nil
}
func (qdh *QueryDecodeHandler) HandleSchema(ctx context.Context, portal *pg.Portal) error {
return qdh.Child.HandleSchema(ctx, portal)
}

4
server/pg_test.go
View file

@ -21,7 +21,7 @@ import (
"testing"
"time"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/logger"
"github.com/molecula/featurebase/v2/pg"
"github.com/molecula/featurebase/v2/pg/pgtest"
@ -33,7 +33,7 @@ func TestPostgresHandler(t *testing.T) {
m := test.RunCommand(t)
defer m.Close()
pgh := server.NewPostgresHandler(m.API, logger.NewLogfLogger(t))
pgh := server.NewPostgresHandler(m.API, logger.NewLogfLogger(t), server.SqlV1)
m.MustCreateIndex(t, "i", pilosa.IndexOptions{TrackExistence: true})
m.MustCreateField(t, "i", "set")

4
server/server.go
View file

@ -40,7 +40,7 @@ import (
"golang.org/x/sync/errgroup"
"github.com/molecula/featurebase/v2"
pilosa "github.com/molecula/featurebase/v2"
"github.com/molecula/featurebase/v2/boltdb"
"github.com/molecula/featurebase/v2/encoding/proto"
petcd "github.com/molecula/featurebase/v2/etcd"
@ -265,7 +265,7 @@ func (m *Command) Start() (err error) {
}
tlsConf = conf
}
m.pgserver = NewPostgresServer(m.API, m.logger, tlsConf)
m.pgserver = NewPostgresServer(m.API, m.logger, tlsConf, SqlVersion(m.Config.Postgres.SqlVersion))
m.pgserver.s.StartupTimeout = time.Duration(m.Config.Postgres.StartupTimeout)
m.pgserver.s.ReadTimeout = time.Duration(m.Config.Postgres.ReadTimeout)
m.pgserver.s.WriteTimeout = time.Duration(m.Config.Postgres.WriteTimeout)

4
server_internal_test.go
View file

@ -19,6 +19,7 @@ import (
"testing"
"time"
"github.com/molecula/featurebase/v2/storage"
"github.com/molecula/featurebase/v2/testhook"
)
@ -45,8 +46,9 @@ func TestMonitorAntiEntropyZero(t *testing.T) {
if err != nil {
t.Fatalf("getting temp dir: %v", err)
}
cfg := &storage.Config{FsyncEnabled: false, Backend: storage.DefaultBackend}
s, err := NewServer(OptServerDataDir(td),
OptServerAntiEntropyInterval(0))
OptServerAntiEntropyInterval(0), OptServerStorageConfig(cfg))
if err != nil {
t.Fatalf("making new server: %v", err)
}

6
sql/mapper.go
View file

@ -25,6 +25,8 @@ import (
const (
SQLTypeSelect = "select"
SQLTypeShow = "show"
SQLTypeSet = "set"
SQLTypeBegin = "begin"
SQLTypeEmpty = ""
)
@ -101,6 +103,10 @@ func (m *Mapper) MapSQL(sql string) (*MappedSQL, error) {
}
case *sqlparser.Show:
sqlType = SQLTypeShow
case *sqlparser.Set:
sqlType = SQLTypeSet
case *sqlparser.Begin:
sqlType = SQLTypeBegin
}
return &MappedSQL{

200
sql2/ast.go
View file

@ -194,6 +194,26 @@ func StatementSource(stmt Statement) Source {
}
}
// Data types
const (
DataTypeBool = "BOOL"
DataTypeDecimal = "DECIMAL"
DataTypeInt = "INT"
DataTypeSet = "SET"
DataTypeText = "TEXT"
DataTypeTimestamp = "TIMESTAMP"
)
// IsDataTypeValid returns true if typ is a valid data type.
func IsDataTypeValid(typ string) bool {
switch typ {
case DataTypeBool, DataTypeInt, DataTypeDecimal, DataTypeText:
return true
default:
return false
}
}
type Expr interface {
Node
expr()
@ -279,6 +299,49 @@ func cloneExprs(a []Expr) []Expr {
return other
}
// ExprDataType returns the data type for an expression.
func ExprDataType(expr Expr) string {
if expr == nil {
return ""
}
switch expr := expr.(type) {
// Simple type assertions
case *BindExpr, *ExprList, *Ident, *NullLit, *Raise:
return ""
case *BlobLit, *StringLit:
return DataTypeText
case *BoolLit, *Exists, *Range:
return DataTypeBool
case *NumberLit:
return DataTypeInt
// Complex type assertions
case *BinaryExpr:
return ExprDataType(expr.X)
case *Call:
return DataTypeInt // TODO: May be different for some aggregations
case *CaseExpr:
if len(expr.Blocks) > 0 {
return ExprDataType(expr.Blocks[0].Body)
} else if expr.ElseExpr != nil {
return ExprDataType(expr.ElseExpr)
}
return ""
case *CastExpr:
return "" // TODO: Inspect expr.Type.Name
case *ParenExpr:
return ExprDataType(expr.X)
case *QualifiedRef:
return expr.DataType
case *UnaryExpr:
return ExprDataType(expr.X)
default:
panic(fmt.Sprintf("invalid expr type: %T", expr))
}
}
// ExprString returns the string representation of expr.
// Returns a blank string if expr is nil.
func ExprString(expr Expr) string {
@ -288,6 +351,119 @@ func ExprString(expr Expr) string {
return expr.String()
}
// ExprTableName returns the name of the table referenced in an expression.
// Returns ok as false if more than one table referenced. Returns a blank string
// if no tables are referenced.
func ExprTableName(expr Expr) (table string, ok bool) {
switch expr := expr.(type) {
case *BindExpr, *BlobLit, *BoolLit, *Ident, *NullLit, *NumberLit, *StringLit:
return "", true
case *BinaryExpr:
x, ok := ExprTableName(expr.X)
if !ok {
return "", false
}
y, ok := ExprTableName(expr.Y)
if !ok {
return "", false
}
if x == "" {
return y, true
} else if y == "" {
return x, true
} else if x == y {
return x, true
}
return "", false
case *Call:
for _, arg := range expr.Args {
tbl, ok := ExprTableName(arg)
if !ok || (table != "" && tbl != table) {
return "", false
}
table = tbl
}
return table, true
case *CaseExpr:
tbl, ok := ExprTableName(expr.Operand)
if !ok || (table != "" && tbl != table) {
return "", false
}
table = tbl
tbl, ok = ExprTableName(expr.ElseExpr)
if !ok || (table != "" && tbl != table) {
return "", false
}
table = tbl
for _, blk := range expr.Blocks {
tbl, ok := ExprTableName(blk.Condition)
if !ok || (table != "" && tbl != table) {
return "", false
}
table = tbl
tbl, ok = ExprTableName(blk.Body)
if !ok || (table != "" && tbl != table) {
return "", false
}
table = tbl
}
return table, true
case *CastExpr:
return ExprTableName(expr.X)
case *Exists:
return "", false // TODO
case *ExprList:
for _, e := range expr.Exprs {
tbl, ok := ExprTableName(e)
if !ok || (table != "" && tbl != table) {
return "", false
}
table = tbl
}
return table, true
case *ParenExpr:
return ExprTableName(expr.X)
case *QualifiedRef:
return expr.Table.Name, true
case *Raise:
return "", true
case *Range:
tbl, ok := ExprTableName(expr.X)
if !ok || (table != "" && tbl != table) {
return "", false
}
table = tbl
tbl, ok = ExprTableName(expr.Y)
if !ok || (table != "" && tbl != table) {
return "", false
}
table = tbl
return table, true
case *UnaryExpr:
return ExprTableName(expr.X)
default:
return "", false
}
}
// SplitExprTree splits apart expr so it is a list of all AND joined expressions.
// For example, the expression "A AND B AND (C OR (D AND E))" would be split into
// a list of "A", "B", "C OR (D AND E)".
@ -1811,6 +1987,9 @@ type QualifiedRef struct {
Dot Pos // position of dot
Star Pos // position of * (result column only)
Column *Ident // column name
// Set by the planner; not at parse-time
DataType string
}
// IsAggregate returns false.
@ -2935,6 +3114,23 @@ func (s *SelectStatement) IsAggregate() bool {
return false
}
// HasWildcard returns true any result column contains a wildcard (STAR).
func (s *SelectStatement) HasWildcard() bool {
for _, col := range s.Columns {
// Unqualified wildcard.
if col.Star.IsValid() {
return true
}
// Table-qualified wildcard.
if ref, ok := col.Expr.(*QualifiedRef); ok && ref.Star.IsValid() {
return true
}
}
return false
}
// String returns the string representation of the statement.
func (s *SelectStatement) String() string {
var buf bytes.Buffer
@ -3061,10 +3257,12 @@ func (c *ResultColumn) Name() string {
}
switch expr := c.Expr.(type) {
case *Call:
return strings.ToLower(IdentName(expr.Name))
case *Ident:
return IdentName(expr)
case *QualifiedRef:
return expr.String()
return IdentName(expr.Column)
default:
return ""
}

4
sql2/ast_test.go
View file

@ -1143,14 +1143,14 @@ func AssertNodeStringerPanic(tb testing.TB, node sql.Node, msg string) {
func StripPos(root sql.Node) sql.Node {
zero := reflect.ValueOf(sql.Pos{})
_ = sql.Walk(sql.VisitFunc(func(node sql.Node) error {
_, _ = sql.Walk(sql.VisitFunc(func(node sql.Node) (sql.Node, error) {
value := reflect.Indirect(reflect.ValueOf(node))
for i := 0; i < value.NumField(); i++ {
if field := value.Field(i); field.Type() == zero.Type() {
field.Set(zero)
}
}
return nil
return node, nil
}), root)
return root
}

12
sql2/scanner.go
View file

@ -104,6 +104,11 @@ func (s *Scanner) Scan() (pos Pos, token Token, lit string) {
case '+':
return pos, PLUS, "+"
case '-':
if s.peek() == '-' {
s.read()
s.skipComment()
continue
}
return pos, MINUS, "-"
case '*':
return pos, STAR, "*"
@ -117,6 +122,13 @@ func (s *Scanner) Scan() (pos Pos, token Token, lit string) {
}
}
// skipComment reads all characters until the end of the line or EOF.
func (s *Scanner) skipComment() {
for ch := s.peek(); ch != '\n' && ch != -1; ch = s.peek() {
s.read()
}
}
func (s *Scanner) scanUnquotedIdent(pos Pos, prefix string) (Pos, Token, string) {
assert(isUnquotedIdent(s.peek()))

3
sql2/scanner_test.go
View file

@ -38,6 +38,9 @@ func TestScanner_Scan(t *testing.T) {
t.Run("StartingX", func(t *testing.T) {
AssertScan(t, `xyz`, sql.IDENT, `xyz`)
})
t.Run("WithComment", func(t *testing.T) {
AssertScan(t, "-- this is a comment\n\n-- more comments\nfoo", sql.IDENT, `foo`)
})
})
t.Run("KEYWORD", func(t *testing.T) {

785
sql2/walk.go
View file

File diff suppressed because it is too large Load diff

4
syswrap/mmap.go
View file

@ -20,6 +20,7 @@ import (
"sync"
"sync/atomic"
"syscall"
"strings"
"github.com/pkg/errors"
)
@ -56,6 +57,9 @@ func Mmap(fd int, offset int64, length int, prot int, flags int) (data []byte, e
data, err = syscall.Mmap(fd, offset, length, prot, flags)
if err != nil {
atomic.AddUint64(&mapCount, ^uint64(0)) // decrement
if strings.Contains(err.Error(), "cannot allocate memory") {
err = errors.New("mmap 'cannot allocate memory' — please see the troubleshooting how-to in the FeatureBase docs.")
}
}
return data, err
}

2
test/cluster.go
View file

@ -607,7 +607,7 @@ func prependTestServerOpts(opts []server.CommandOption) []server.CommandOption {
pilosa.OptServerNodeDownRetries(5, 100*time.Millisecond),
pilosa.OptServerStorageConfig(&storage.Config{
Backend: pilosa.CurrentBackendOrDefault(),
FsyncEnabled: true,
FsyncEnabled: false,
}),
),
}

19
test/disco.go
View file

@ -111,6 +111,7 @@ func GetPortsGenConfigs(tb testing.TB, nodes []*Command) error {
LPeerSocket: []*net.TCPListener{peerListener},
LClientSocket: []*net.TCPListener{clientListener},
BootstrapTimeout: 50 * time.Millisecond,
UnsafeNoFsync: true,
}
peerUrls[i] = fmt.Sprintf("%s=%s", name, peerURL)
}
@ -144,14 +145,16 @@ func GenPortsConfig(tb testing.TB, ports []Ports) []*server.Config {
BindGRPC: fmt.Sprintf(":%d", ports[i].Grpc),
GRPCListener: ports[i].LsnG,
Etcd: etcd.Options{
Dir: discoDir,
LClientURL: lClientURL,
AClientURL: lClientURL,
LPeerURL: lPeerURL,
APeerURL: lPeerURL,
HeartbeatTTL: 5,
LPeerSocket: []*net.TCPListener{lsnP},
LClientSocket: []*net.TCPListener{lsnC},
Dir: discoDir,
LClientURL: lClientURL,
AClientURL: lClientURL,
LPeerURL: lPeerURL,
APeerURL: lPeerURL,
HeartbeatTTL: 5,
LPeerSocket: []*net.TCPListener{lsnP},
LClientSocket: []*net.TCPListener{lsnC},
BootstrapTimeout: 50 * time.Millisecond,
UnsafeNoFsync: true,
},
}
cfgs[i].Cluster.Name = clusterName

5
test/holder.go
View file

@ -38,7 +38,10 @@ func NewHolder(tb testing.TB) *Holder {
panic(err)
}
h := &Holder{Holder: pilosa.NewHolder(path, nil)}
cfg := pilosa.DefaultHolderConfig()
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
h := &Holder{Holder: pilosa.NewHolder(path, cfg)}
return h
}

5
test/index.go
View file

@ -33,7 +33,10 @@ func newIndex(tb testing.TB) *Index {
if err != nil {
panic(err)
}
h := pilosa.NewHolder(path, pilosa.DefaultHolderConfig())
cfg := pilosa.DefaultHolderConfig()
cfg.StorageConfig.FsyncEnabled = false
cfg.RBFConfig.FsyncEnabled = false
h := pilosa.NewHolder(path, cfg)
testhook.Cleanup(tb, func() {
h.Close()
})

34
testBackupRestore.sh
View file

@ -13,19 +13,19 @@ STATUS=$STATUS timeout -s TERM $TIMEOUT bash -c \
done;'
echo "NOW DO STUFF"
datagen --source kitchensink_keyed -e 9999 --pilosa.index sink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101
before=$(/pilosa chksum --host pilosa0:10101)
/pilosa backup -o backupdir --host pilosa0:10101
before=$(/featurebase chksum --host pilosa0:10101)
/featurebase backup -o backupdir --host pilosa0:10101
curl -X DELETE -s pilosa0:10101/index/sink
/pilosa restore -s backupdir --host pilosa0:10101
after=$(/pilosa chksum --host pilosa0:10101)
/featurebase restore -s backupdir --host pilosa0:10101
after=$(/featurebase chksum --host pilosa0:10101)
if [ "$before" = "$after" ]; then
echo "PASS Cluster"
else
echo "FAIL Single"
exit 1
fi
/pilosa restore -s backupdir --host pilosax:10101
single=$(/pilosa chksum --host pilosax:10101)
/featurebase restore -s backupdir --host pilosax:10101
single=$(/featurebase chksum --host pilosax:10101)
if [ "$before" = "$single" ]; then
echo "PASS Single"
exit 0
@ -33,3 +33,25 @@ else
echo "FAIL Single"
exit 1
fi
datagen --source texas_health -e 9999 --pilosa.index newsink --pilosa.batch-size 10000 --pilosa.hosts pilosa0:10101
before=$(/featurebase chksum --host pilosa0:10101)
/featurebase backup -o newbackupdir --host pilosa0:10101 --index newsink
curl -X DELETE -s pilosa0:10101/index/newsink
/featurebase restore -s newbackupdir --host pilosa0:10101
after=$(/featurebase chksum --host pilosa0:10101)
if [ "$before" = "$after" ]; then
echo "PASS Cluster Table"
else
echo "FAIL Single Table"
exit 1
fi
/featurebase restore -s newbackupdir --host pilosax:10101
single=$(/featurebase chksum --host pilosax:10101)
if [ "$before" = "$single" ]; then
echo "PASS Single Table"
exit 0
else
echo "FAIL Single Table"
exit 1
fi

38
translate.go
View file

@ -23,6 +23,7 @@ import (
"sort"
"sync"
"github.com/molecula/featurebase/v2/ingest"
"github.com/molecula/featurebase/v2/topology"
"github.com/pkg/errors"
)
@ -98,6 +99,39 @@ type TranslateStore interface { // TODO: refactor this interface; readonly shoul
ReadFrom(io.Reader) (int64, error)
}
// This implements ingest's key translator interface, which differs
// slightly because we want to be able to do fast lookups on arbitrary
// IDs which are not necessarily contiguous small values, so the []string
// from TranslateIDs isn't a good fit.
type ingestKeyTranslator struct {
store TranslateStore
}
var _ ingest.KeyTranslator = &ingestKeyTranslator{}
func (i ingestKeyTranslator) TranslateKeys(keys ...string) (map[string]uint64, error) {
return i.store.CreateKeys(keys...)
}
func (i ingestKeyTranslator) TranslateIDs(ids ...uint64) (map[uint64]string, error) {
keys, err := i.store.TranslateIDs(ids)
if err != nil {
return nil, err
}
if len(keys) != len(ids) {
return nil, fmt.Errorf("translating %d id(s), got %d key(s)", len(ids), len(keys))
}
out := make(map[uint64]string, len(keys))
for i, id := range ids {
out[id] = keys[i]
}
return out, nil
}
func newIngestKeyTranslatorFromStore(s TranslateStore) *ingestKeyTranslator {
return &ingestKeyTranslator{store: s}
}
// TranslatorSummary is returned, for example from the boltdb string key translators,
// by calling ComputeTranslatorSummary(). Non-boltdb mocks, etc no-op that method.
type TranslatorSummary struct {
@ -163,7 +197,7 @@ TranslatorSummary{
}
// OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore.
type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int) (TranslateStore, error)
type OpenTranslateStoreFunc func(path, index, field string, partitionID, partitionN int, fsyncEnabled bool) (TranslateStore, error)
// GenerateNextPartitionedID returns the next ID within the same partition.
func GenerateNextPartitionedID(index string, prev uint64, partitionID, partitionN int) uint64 {
@ -373,7 +407,7 @@ var _ OpenTranslateStoreFunc = OpenInMemTranslateStore
// OpenInMemTranslateStore returns a new instance of InMemTranslateStore.
// Implements OpenTranslateStoreFunc.
func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int) (TranslateStore, error) {
func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partitionN int, fsyncEnabled bool) (TranslateStore, error) {
return NewInMemTranslateStore(index, field, partitionID, partitionN), nil
}

2
utils_internal_test.go
View file

@ -117,7 +117,7 @@ func NewTestClusterWithReplication(tb testing.TB, nNodes, nReplicas, partitionN
}
// holder
h := NewHolder(path, nil)
h := NewHolder(path, mustHolderConfig())
// cluster
availableShardFileFlushDuration.Set(100 * time.Millisecond)

4
view.go
View file

@ -300,8 +300,8 @@ func (v *view) Fragment(shard uint64) *fragment {
// allFragments returns a list of all fragments in the view.
func (v *view) allFragments() []*fragment {
v.mu.Lock()
defer v.mu.Unlock()
v.mu.RLock()
defer v.mu.RUnlock()
other := make([]*fragment, 0, len(v.fragments))
for _, fragment := range v.fragments {

2
view_internal_test.go
View file

@ -35,7 +35,7 @@ func mustOpenView(tb testing.TB, index, field, name string) *view {
CacheSize: DefaultCacheSize,
}
h := NewHolder(path, nil)
h := NewHolder(path, mustHolderConfig())
// h needs an *Index so we can call h.Index() and get Index.Txf, in TestView_DeleteFragment
cim := &CreateIndexMessage{