diff --git a/.circleci/config.yml b/.circleci/config.yml index 0f4fdb943..86814df27 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -198,12 +198,24 @@ workflows: resource_class: xlarge requires: - setup + - test: + name: test-txstore-rbf + test_make_target: test-txstore-rbf + resource_class: xlarge + requires: + - setup + - test: + name: test-txstore-rbf_lmdb + test_make_target: test-txstore-rbf_lmdb + resource_class: xlarge + requires: + - setup - test: name: test-shardwidth-22 shard_width: "22" resource_class: large requires: - - setup + - setup - cluster-tests: requires: - setup diff --git a/.gitignore b/.gitignore index 7a41479b9..84fdc73f4 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,6 @@ vendor build *~ lattice +release-pilosa-fsck.*.*.tar.gz +/log.* +/tourna.log.* \ No newline at end of file diff --git a/Makefile b/Makefile index ba3d994a8..058a88847 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race +.PHONY: build check-clean clean build-lattice cover cover-viz default docker docker-build docker-test docker-tag-push generate generate-protoc generate-pql generate-statik gometalinter install install-build-deps install-golangci-lint install-gometalinter install-protoc install-protoc-gen-gofast install-peg install-statik prerelease prerelease-upload release release-build test testv testv-race testvsub testvsub-race test-txstore-rbf_lmdb test-txstore-rbf CLONE_URL=github.com/pilosa/pilosa VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) @@ -55,7 +55,7 @@ testv-race: topt-race testvsub-race # find which test is hung/deadlocked. # testvsub: - set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \ + set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i"; \ cd $$i; pwd; \ go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \ @@ -64,7 +64,7 @@ testvsub: done testvsub-race: - set -e; for i in ctl http pg pql rbf roaring server sql txkey; do \ + set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i -race"; \ cd $$i; pwd; \ go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \ @@ -166,9 +166,14 @@ generate-stringer: generate-pql: require-peg cd pql && peg -inline pql.peg && cd .. -# dunno if protoc-gen-gofast is actually needed here -generate-proto-grpc: require-protoc require-protoc-gen-gofast +generate-proto-grpc: require-protoc require-protoc-gen-go protoc -I proto proto/pilosa.proto --go_out=plugins=grpc:proto + protoc -I proto proto/vdsm/vdsm.proto --go_out=plugins=grpc:proto + # TODO: Modify above commands and remove the below mv if possible. + # See https://go-review.googlesource.com/c/protobuf/+/219298/ for info on --go-opt + # I couldn't get it to work during development - Cody + cp -r proto/github.com/pilosa/pilosa/v2/proto/ proto/ + rm -rf proto/github.com # `go generate` all needed packages generate: generate-protoc generate-statik generate-stringer generate-pql @@ -196,6 +201,9 @@ pilosa-keydump: pilosa-chk: go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) ./cmd/pilosa-chk +pilosa-fsck: + cd ./cmd/pilosa-fsck && make install && make release + # Run Pilosa tests inside Docker container docker-test: docker run --rm -v $(PWD):/go/src/$(CLONE_URL) -w /go/src/$(CLONE_URL) golang:$(GO_VERSION) go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) ./... @@ -386,6 +394,9 @@ install-stringer: install-protoc-gen-gofast: GO111MODULE=off go get -u github.com/gogo/protobuf/protoc-gen-gofast +install-protoc-gen-go: + GO111MODULE=off go get -u github.com/golang/protobuf/protoc-gen-go + install-protoc: @echo This tool cannot automatically install protoc. Please download and install protoc from https://google.github.io/proto-lens/installing-protoc.html @@ -399,3 +410,10 @@ install-gometalinter: GO111MODULE=off go get -u github.com/alecthomas/gometalinter GO111MODULE=off gometalinter --install GO111MODULE=off go get github.com/remyoudompheng/go-misc/deadcode + +test-txstore-rbf: + PILOSA_TXSRC=rbf $(MAKE) testv-race + +test-txstore-rbf_lmdb: + PILOSA_TXSRC=rbf_lmdb $(MAKE) testv-race + diff --git a/api.go b/api.go index b6e8187c5..dfbd33e01 100644 --- a/api.go +++ b/api.go @@ -25,6 +25,8 @@ import ( "io/ioutil" "math" "net/url" + "os" + "path" "sort" "strconv" "strings" @@ -376,37 +378,42 @@ func importWorker(importWork chan importJob) { } } - tx, finisher := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard}) - defer finisher(&err0) + if err := func() (err1 error) { + tx, finisher := j.qcx.GetTx(Txo{Write: writable, Index: j.field.idx, Shard: j.shard}) + defer finisher(&err1) - var doClear bool - switch doAction { - case RequestActionOverwrite: - err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block) - if err != nil { - return errors.Wrap(err, "importing roaring as overwrite") - } - case RequestActionClear: - doClear = true - fallthrough - case RequestActionSet: - fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) - if fileMagic == roaring.MagicNumber { // if pilosa roaring format - err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear) + var doClear bool + switch doAction { + case RequestActionOverwrite: + err := j.field.importRoaringOverwrite(j.ctx, tx, viewData, j.shard, viewName, j.req.Block) if err != nil { - return errors.Wrap(err, "importing pilosa roaring") + return errors.Wrap(err, "importing roaring as overwrite") } - } else { - // must make a copy of data to operate on locally on standard roaring format. - // field.importRoaring changes the standard roaring run format to pilosa roaring - data := make([]byte, len(viewData)) - copy(data, viewData) - err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) + case RequestActionClear: + doClear = true + fallthrough + case RequestActionSet: + fileMagic := uint32(binary.LittleEndian.Uint16(viewData[0:2])) + if fileMagic == roaring.MagicNumber { // if pilosa roaring format + err := j.field.importRoaring(j.ctx, tx, viewData, j.shard, viewName, doClear) + if err != nil { + return errors.Wrap(err, "importing pilosa roaring") + } + } else { + // must make a copy of data to operate on locally on standard roaring format. + // field.importRoaring changes the standard roaring run format to pilosa roaring + data := make([]byte, len(viewData)) + copy(data, viewData) + err := j.field.importRoaring(j.ctx, tx, data, j.shard, viewName, doClear) - if err != nil { - return errors.Wrap(err, "importing standard roaring") + if err != nil { + return errors.Wrap(err, "importing standard roaring") + } } } + return nil + }(); err != nil { + return err } } return nil @@ -774,12 +781,85 @@ func (api *API) Hosts(ctx context.Context) []*Node { return api.cluster.Nodes() } +func (api *API) HostStates(ctx context.Context) map[string]string { + span, _ := tracing.StartSpanFromContext(ctx, "API.HostStates") + defer span.Finish() + return api.cluster.AllNodeStates() +} + // Node gets the ID, URI and coordinator status for this particular node. func (api *API) Node() *Node { node := api.server.node() return &node } +// Usage gets the disk usage per index +func (api *API) Usage() (map[string]int64, int64, error) { + indexSizes := make(map[string]int64) + var totalSize int64 + + dirName, err := expandDirName(api.server.dataDir) + if err != nil { + return indexSizes, totalSize, errors.Wrap(err, "expanding data directory") + } + dir, err := os.Open(dirName) + if err != nil { + return indexSizes, totalSize, errors.Wrap(err, "opening data directory") + } + defer dir.Close() + + files, err := dir.Readdir(-1) + if err != nil { + return indexSizes, totalSize, errors.Wrap(err, "reading data directory") + } + + for _, file := range files { + if !file.IsDir() { + continue + } + if api.holder.Txf().IsTxDatabasePath(file.Name()) { + continue + } + fullName := path.Join(dirName, file.Name()) + indexSizes[file.Name()], err = diskUsage(fullName) + if err != nil { + break + } + totalSize += indexSizes[file.Name()] + } + + return indexSizes, totalSize, nil +} + +func diskUsage(fname string) (int64, error) { + var size int64 + + dir, err := os.Open(fname) + if err != nil { + return 0, errors.Wrap(err, "opening data subdirectory") + } + defer dir.Close() + + files, err := dir.Readdir(-1) + if err != nil { + return 0, errors.Wrap(err, "reading data subdirectory") + } + + for _, file := range files { + if file.IsDir() { + sz, err := diskUsage(path.Join(fname, file.Name())) + if err != nil { + return 0, err + } + size += sz + } else { + size += file.Size() + } + } + + return size, nil +} + // RecalculateCaches forces all TopN caches to be updated. // This is done internally within a TopN query, but a user may want to do it ahead of time? func (api *API) RecalculateCaches(ctx context.Context) error { @@ -1667,6 +1747,14 @@ func (api *API) State() string { return api.cluster.State() } +// ClusterName returns the cluster name. +func (api *API) ClusterName() string { + if api.cluster.Name == "" { + return api.cluster.id + } + return api.cluster.Name +} + // Version returns the Pilosa version. func (api *API) Version() string { return strings.TrimPrefix(Version, "v") @@ -1692,6 +1780,7 @@ func (api *API) Info() serverInfo { CPUType: si.CPUModel(), Memory: mem, TxSrc: api.holder.txf.TxType(), + ReplicaN: api.cluster.ReplicaN, } } @@ -1938,6 +2027,7 @@ func (api *API) TranslateFieldDB(ctx context.Context, indexName, fieldName strin type serverInfo struct { ShardWidth uint64 `json:"shardWidth"` + ReplicaN int `json:"replicaN"` Memory uint64 `json:"memory"` CPUType string `json:"cpuType"` CPUPhysicalCores int `json:"cpuPhysicalCores"` diff --git a/boltdb/translate.go b/boltdb/translate.go index 78e81ed7a..b5a0e5651 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -17,9 +17,12 @@ package boltdb import ( "bytes" "context" + "fmt" "io" + "io/ioutil" "os" "path/filepath" + "sort" "sync" "time" @@ -27,8 +30,13 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pkg/errors" "github.com/zeebo/blake3" + + "runtime/pprof" ) +var _ = ioutil.TempFile +var _ = pprof.StartCPUProfile + var ( // ErrTranslateStoreClosed is returned when reading from an TranslateEntryReader // and the underlying store is closed. @@ -90,6 +98,10 @@ type TranslateStore struct { Path string } +func (s *TranslateStore) GetStorePath() string { + return s.Path +} + // NewTranslateStore returns a new instance of TranslateStore. func NewTranslateStore(index, field string, partitionID, partitionN int) *TranslateStore { return &TranslateStore{ @@ -104,6 +116,15 @@ func NewTranslateStore(index, field string, partitionID, partitionN int) *Transl // Open opens the translate file. func (s *TranslateStore) Open() (err error) { + + // add the path to the problem database if we panic handling it. + defer func() { + r := recover() + if r != nil { + panic(fmt.Sprintf("pilosa/boltdb/TranslateStore.Open(s.Path='%v') panic with '%v'", s.Path, r)) + } + }() + if err := os.MkdirAll(filepath.Dir(s.Path), 0777); err != nil { return errors.Wrapf(err, "mkdir %s", filepath.Dir(s.Path)) } else if s.db, err = bolt.Open(s.Path, 0666, &bolt.Options{Timeout: 1 * time.Second}); err != nil { @@ -485,7 +506,7 @@ func findKeyByID(bkt *bolt.Bucket, id uint64) string { return string(boltKey) } -func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) { +func (s *TranslateStore) ComputeTranslatorSummaryRows() (sum *pilosa.TranslatorSummary, err error) { sum = &pilosa.TranslatorSummary{} hasher := blake3.New() @@ -524,3 +545,787 @@ func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSumma sum.Checksum = string(buf[:]) return sum, nil } + +func (s *TranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *pilosa.Topology) (sum *pilosa.TranslatorSummary, err error) { + sum = &pilosa.TranslatorSummary{} + hasher := blake3.New() + + if partitionID != s.partitionID { + panic(fmt.Sprintf("inconsistent partitionID arg %v with TranslateStore.paritionID %v", partitionID, s.partitionID)) + } + firstPrimary := topo.PrimaryNodeIndex(partitionID) + + err = s.db.View(func(tx *bolt.Tx) error { + + bkt := tx.Bucket(bucketKeys) // key -> id + if bkt == nil { + panic("bucketKeys not found") + } + + cur := bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + input := append(k, v...) + //vv("55555 ComputeTranslatorSummaryCols(partitionID=%v, path='%v'), k='%v', v=%x", partitionID, s.Path, string(k), v) + _, _ = hasher.Write(input) + sum.KeyCount++ + } + + bkt = tx.Bucket(bucketIDs) // id -> key + if bkt == nil { + panic("bucketIDs not found") + } + + cur = bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + + // should the primary be the same for each key in this partition? + id := btou64(k) + shard := id / pilosa.ShardWidth + + ks := string(v) + primary := topo.GetPrimaryForColKeyTranslation(s.index, ks) + if firstPrimary < 0 { + firstPrimary = primary + } else { + if primary != firstPrimary { + panic(fmt.Sprintf("s.index='%v' primary (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v; topo='%v'", s.index, primary, firstPrimary, ks, id, shard, partitionID, topo.String())) + } + } + + // Verify the invariant that the primaries agree. Just a sanity check. + primaryForShard := topo.GetPrimaryForShardReplication(s.index, shard) + if primaryForShard != firstPrimary { + panic(fmt.Sprintf("primaryForShard (%v) != firstPrimary (%v); key='%v', id=%v, shard=%v; partitionID=%v", primaryForShard, firstPrimary, ks, id, shard, partitionID)) + } + + input := append(k, v...) + //vv("55555 ComputeTranslatorSummaryCols(partitionID=%v, path='%v'), idBucket id=%x key='%v'", partitionID, s.Path, id, ks) + _, _ = hasher.Write(input) + sum.IDCount++ + } + return nil + }) + if err != nil { + return nil, err + } + + sum.PrimaryNodeIndex = firstPrimary + + var buf [16]byte + _, _ = hasher.Digest().Read(buf[0:]) + sum.Checksum = string(buf[:]) + return sum, nil +} + +func (s *TranslateStore) KeyWalker(walk func(key string, col uint64)) error { + return s.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(bucketKeys) + if bkt == nil { + panic("bucketKeys not found") + } + cur := bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + walk(string(k), btou64(v)) + } + return nil + }) +} +func (s *TranslateStore) IDWalker(walk func(key string, col uint64)) error { + return s.db.View(func(tx *bolt.Tx) error { + bkt := tx.Bucket(bucketIDs) + if bkt == nil { + panic("bucketIDs not found") + } + cur := bkt.Cursor() + for k, v := cur.First(); k != nil; k, v = cur.Next() { + walk(string(v), btou64(k)) + } + return nil + }) +} + +// call s.notifyWrite() when done +func (s *TranslateStore) SetFwdRevMaps(tx *bolt.Tx, fwd map[string]uint64, rev map[uint64]string) (err error) { + + localTx := false + if tx == nil { + localTx = true + tx, err = s.db.Begin(true) + if err != nil { + return err + } + defer func() { + _ = tx.Rollback() + }() + } + + // reinitialize buckets + err = tx.DeleteBucket(bucketKeys) + if err != nil { + return err + } + err = tx.DeleteBucket(bucketIDs) + if err != nil { + return err + } + if _, err := tx.CreateBucketIfNotExists(bucketKeys); err != nil { + return err + } else if _, err := tx.CreateBucketIfNotExists(bucketIDs); err != nil { + return err + } + + key2id := tx.Bucket(bucketKeys) + for k, v := range fwd { + err := key2id.Put([]byte(k), u64tob(v)) + if err != nil { + return err + } + } + id2key := tx.Bucket(bucketIDs) + for k, v := range rev { + err := id2key.Put(u64tob(k), []byte(v)) + if err != nil { + return err + } + } + if localTx { + return tx.Commit() + } + return nil +} + +func (s *TranslateStore) GetFwdRevMaps(tx *bolt.Tx) (fwd map[string]uint64, rev map[uint64]string, err error) { + fwd = make(map[string]uint64) + rev = make(map[uint64]string) + + key2id := tx.Bucket(bucketKeys) + + err = key2id.ForEach(func(k, v []byte) error { + fwd[string(k)] = btou64(v) + return nil + }) + if err != nil { + return + } + + id2key := tx.Bucket(bucketIDs) + err = id2key.ForEach(func(k, v []byte) error { + rev[btou64(k)] = string(v) + return nil + }) + return +} + +//var vv = pilosa.VV + +// helpers for repair + +// muint64 holds multiple unit64 +type muint64 struct { + slc []uint64 +} + +func (m *muint64) String() (s string) { + for _, e := range m.slc { + s += fmt.Sprintf("%x, ", e) + } + return +} + +// mstring holds multiple strings +type mstring struct { + slc []string +} + +func (m *mstring) String() (s string) { + for _, e := range m.slc { + s += e + "," + } + return +} + +func addToProblemKeys(problemKeys map[string]*muint64, k string, v uint64, noValue bool) { + mu, already := problemKeys[k] + if !already { + mu = &muint64{} + problemKeys[k] = mu + } + if !noValue { + mu.slc = append(mu.slc, v) + } +} +func addToProblemIDs(problemIDs map[uint64]*mstring, k uint64, v string, noValue bool) { + mu, already := problemIDs[k] + if !already { + mu = &mstring{} + problemIDs[k] = mu + } + if !noValue { + mu.slc = append(mu.slc, v) + } +} + +// only actually apply the fixes if applyKeyRepairs is true. +// if anything changed, return changed == true. +func (s *TranslateStore) RepairKeys(topo *pilosa.Topology, verbose, applyKeyRepairs bool) (changed bool, err error) { + // strategy: get the full set of keys; the domain keys from + // the forward key->id mapping, and the range keys from the reverse id->key mapping. + // Then march through them and make sure they are mapped correctly. + // At the moment we do try to reuse dangling IDs instead of making + // new ones. This might not always be possible, but we hope for + // now that it suffices b/c it minimizes the amount of fragment + // re-write we may have to do. + + /* + // ============ profiling =============== + fd, err := ioutil.TempFile(".", "cpu.prof") + if err != nil { + panic(err) + } + _ = pprof.StartCPUProfile(fd) + defer func() { + pprof.StopCPUProfile() + fd.Close() + }() + // ============ end profiling =============== + */ + + tx, err := s.db.Begin(true) + if err != nil { + return false, err + } + defer func() { + _ = tx.Rollback() + }() + fwd, rev, err := s.GetFwdRevMaps(tx) + if err != nil { + return false, err + } + + // place to store the correct stuff. + // + // fwd2, rev2: new, repaired versions. + // INVAR: they only contain (correct) invertible mappings. + fwd2 := make(map[string]uint64) + rev2 := make(map[uint64]string) + + // and a place to store the problems. + problemKeys := make(map[string]*muint64) + problemIDs := make(map[uint64]*mstring) + +fwdscan: + for k, v := range fwd { + _, already := fwd2[k] + if already { + // k has already been repaired. don't worry about further. + continue fwdscan + } else { + // if its already invertible, then just keep it, no need to repair it + + // INVAR: k is not in fwd2 (at least not yet). + rkey, ok := rev[v] + if !ok { + // k -> v -> X + addToProblemIDs(problemIDs, v, k, false) + addToProblemKeys(problemKeys, k, v, false) + continue fwdscan + } + if rkey == k { + // yay. a good, invertible, mapping. no repair needed. + if k == "" { + panic("bad empty key") + } + fwd2[k] = v + rev2[v] = k + continue fwdscan + } + + // some kind of problem. + // what kind? + // Define problemKey as: 2nd key mapping to id already in fwd2. + // Define problemID as: 2nd ID mapping to key already in fwd2. + + // k -> v -> rkey, and rkey != k. + v2, ok := fwd[rkey] + if ok { + // k -> v -> rkey -> v2, where v2 ?= v + if v2 == v { + // k -> v -> rkey -> v + // just have a problemKey in k. + + if rkey == "" { + panic("bad empty rkey") + } + fwd2[rkey] = v + rev2[v] = rkey + addToProblemKeys(problemKeys, k, v, true) + continue fwdscan + } + // this is i = 1 test case. :) + + // k -> v -> rkey -> v2, where v != v2, and k != rkey. + addToProblemKeys(problemKeys, k, v, false) + addToProblemIDs(problemIDs, v, rkey, false) + continue fwdscan + } else { + // k -> v -> rkey -> X(nil), and rkey != k. + addToProblemKeys(problemKeys, k, v, false) + addToProblemKeys(problemKeys, rkey, 0, true) + addToProblemIDs(problemIDs, v, rkey, false) + } + } + } +revscan: + for id, key := range rev { + k1, already := rev2[id] + _ = k1 + if already { + // fine, already there. + continue + } + + // if its already invertible, keep it. + rid, ok := fwd[key] + if !ok { + // id -> key -> X + addToProblemKeys(problemKeys, key, 0, true) + addToProblemIDs(problemIDs, id, key, false) + continue revscan + } + if rid == id { + // id -> key -> id. good. but should have been added to fwd2/rev2 above. + panic("should have been added to fwd2/rev2 above!") + + } else { + // id -> key -> rid, where id != rid + // so rid -> ? + keyr, ok := rev[rid] + if !ok { + // id -> key -> rid -> X, where id != rid + addToProblemKeys(problemKeys, key, rid, false) + addToProblemKeys(problemKeys, key, id, false) + addToProblemIDs(problemIDs, rid, key, false) + continue revscan + } + if keyr == key { + // id -> key -> rid -> key. So rid is correct and id is dangling. + // + // Heuristic: ASSUME here, that the 2 consistent links key->rid->key are correct, + // and that the single id -> key is in the wrong. This DOESN'T HAVE + // TO BE THE CASE. + + if key == "" { + panic("bad empty key") + } + rev2[rid] = key + fwd2[key] = rid + addToProblemIDs(problemIDs, id, "", true) + } else { + // this is test case i = 0. Must handle it. + + // id -> key -> rid -> keyr, id != rid, keyr != key. + id2, ok := fwd[keyr] + if ok && id2 == rid { + // id -> key -> rid -> keyr -> rid, id != rid, keyr != key. + // so rid -> keyr -> rid is good. + + if keyr == "" { + panic("bad empty keyr") + } + fwd2[keyr] = rid + rev2[rid] = keyr + // and id -> key -> rid is bad, b/c id != rid. + addToProblemKeys(problemKeys, key, rid, false) + addToProblemIDs(problemIDs, id, key, false) + continue revscan + } + // one of these 3 cases holds. all have the same treatment. + // 1) id2 == id: id -> key -> rid -> keyr -> id2; id != rid, rid != id2, keyr != key. + // 2) id2 != id: id -> key -> rid -> keyr -> id2; id != rid, rid != id2, id2 != id, keyr != key. + // 3) !ok: id -> key -> rid -> keyr -> X, id != rid, keyr != key. + addToProblemIDs(problemIDs, id, key, false) + addToProblemKeys(problemKeys, key, rid, false) + addToProblemIDs(problemIDs, rid, keyr, false) + } + } + + } + //vv("problemKeys = '%v'", problemKeys) + //vv("problemIDs = '%v'", problemIDs) + + newIDs := make(map[uint64]bool) + + // assign new IDs to any problemKeys; but first + // try to reuse already allocated IDs that are just dangling. +loopProblemKeys: + for key, ids := range problemKeys { + // first try a minor repair, maybe it was just mssing from rev + // and we can avoid allocate another id. + + // sanity check + v2, already := fwd2[key] + if already { + panic(fmt.Sprintf("should not get here since fwd2 is only correct invertibles: key='%v', v2='%x'", key, v2)) + } + // INVAR: we have no correct mapping for key in fwd2. + + // treat the danglers as "suggestions" for the correction. + for k, id := range ids.slc { + _ = k + _, already = rev2[id] + if !already { + // is this correct? + // id is not in rev2, and key is not in fwd2. + // therefore, we can add them both and maintain consistency. + + //vv("add %v to fwd2", key) + if key == "" { + panic("bad empty key") + } + fwd2[key] = id + rev2[id] = key + continue loopProblemKeys + } + } + // INVAR: key -> ? don't know. We didn't find a usable suggestion for the id. + + // yes, we get here. We have key. We are looking for a suitable id for it. + + // can we get a usable id from the problemIDs? + found := false + suggestions: + for idp, mkeyp := range problemIDs { + for _, candk := range mkeyp.slc { + //vv("checking problemIDs, ipd=%x, candk='%v'; candk==key is %v", idp, candk, candk == key) + if candk == key { + // we have a suggestion from problemIDs that idp might work, doing key -> idp. + // Validate that this is possible. + k2, already := rev2[idp] + _ = k2 + if already { + //vv("idp is already in rev2: idp=%v, k2=%v", idp, k2) + continue suggestions + } + // idp works. put it in the correct set. + if key == "" { + panic("bad empty key") + } + rev2[idp] = key + fwd2[key] = idp + found = true + break suggestions + } + } + } + if !found { + id2 := pilosa.GenerateNextPartitionedID(s.index, maxID(tx), s.partitionID, s.partitionN) + //vv("could not minor repair, allocating new id2 = %v instead", id2) + newIDs[id2] = true + + if key == "" { + panic("bad empty key") + } + fwd2[key] = id2 + rev2[id2] = key + } + } // end problemKeys + + //for id, keys := range problemIDs { + //} + + if verbose { + reportIfGainedOrLostIDs(s, fwd, fwd2, rev, rev2, newIDs) + reportIfGainedOrLostKeys(s, fwd, fwd2, rev, rev2) + } + + adds, changes, changeIDs, err := makeStringKeyChanges(verbose, applyKeyRepairs, tx, s, topo, fwd, fwd2, rev, rev2, newIDs) + if err != nil { + return false, err + } + _, _, _ = adds, changes, changeIDs + + //vv("changedIDs = '%#v'", changeIDs) + if len(adds) > 0 || len(changes) > 0 || len(changeIDs) > 0 || len(newIDs) > 0 { + changed = true + } + + //vv("newIDs = '%#v'", newIDs) + //vv("fwd2 = '%#v'", fwd2) + //vv("rev2 = '%#v'", rev2) + + err = tx.Commit() + if err == nil { + s.notifyWrite() + } + return changed, err +} + +func reportIfGainedOrLostIDs(s *TranslateStore, fwd, fwd2 map[string]uint64, rev, rev2 map[uint64]string, newIDs map[uint64]bool) { + // get all IDs ever mentioned + before := make(map[uint64]bool) + after := make(map[uint64]bool) + for _, id := range fwd { + before[id] = true + } + for _, id := range fwd2 { + if !newIDs[id] { + after[id] = true + } + } + for id := range rev { + before[id] = true + } + for id := range rev2 { + if !newIDs[id] { + after[id] = true + } + } + nb := len(before) + na := len(after) + if nb != na { + fmt.Printf("# needs-repair: Num ID before %v != Num ID after %v, for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v\n", nb, na, s.Path, len(fwd), len(rev), len(fwd2), len(rev2)) + } + if len(newIDs) > 0 { + fmt.Printf("# needs-repair: adding newIDs '%#v', for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v\n", newIDs, s.Path, len(fwd), len(rev), len(fwd2), len(rev2)) + } +} +func reportIfGainedOrLostKeys(s *TranslateStore, fwd, fwd2 map[string]uint64, rev, rev2 map[uint64]string) { + // get all IDs ever mentioned + nb := len(fwd) + na := len(fwd2) + if nb != na { + diffAB := mapDiffStrings(fwd, fwd2) + diffBA := mapDiffStrings(fwd2, fwd) + + fmt.Printf("# needs-repair: Num Keys before != Num Keys after, for boltdb = '%v'. before counts(fwd/rev) = %v/%v. after repair counts(fwd2/rev2) = %v/%v. fwd - fwd2 = '%#v'; fwd2-fwd = '%#v'\n", s.Path, len(fwd), len(rev), len(fwd2), len(rev2), diffAB, diffBA) + } +} + +// return A - B +func mapDiffStrings(mapA, mapB map[string]uint64) (r []string) { + for a := range mapA { + _, ok := mapB[a] + if !ok { + r = append(r, a) + } + } + sort.Strings(r) + return +} + +type BeforeAfterKeyChange struct { + BeforeID uint64 + AfterID uint64 +} + +type BeforeAfterIDChange struct { + IsDelete bool + IsAdd bool + BeforeString string + AfterString string +} + +// do the minimal state update. +// fwd2 is the "after" map, all string keys repaired. +func makeStringKeyChanges( + verbose bool, + applyKeyRepairs bool, + tx *bolt.Tx, + s *TranslateStore, + topo *pilosa.Topology, + fwd, fwd2 map[string]uint64, + rev, rev2 map[uint64]string, + newIDs map[uint64]bool, +) ( + adds map[string]uint64, + changeKeys map[string]*BeforeAfterKeyChange, + changeIDs map[uint64]*BeforeAfterIDChange, + err error, +) { + //vv("makeStringKeyChanges called") + + //vv("fwd2 = '%#v'", fwd2) + //vv("rev2 = '%#v'", rev2) + //vv("fwd = '%#v'", fwd) + //vv("rev = '%#v'", rev) + + var action string + if applyKeyRepairs { + action = "applying " + } + + // addition of string key + adds = make(map[string]uint64) + + // change of the mapping of key -> id. + changeKeys = make(map[string]*BeforeAfterKeyChange) + + // changes to bucketIDs + changeIDs = make(map[uint64]*BeforeAfterIDChange) + + localTx := false + if applyKeyRepairs && tx == nil { + localTx = true + tx, err = s.db.Begin(true) + if err != nil { + return + } + defer func() { + //vv("tx.Rollback happening") + _ = tx.Rollback() + }() + } + + key2id := tx.Bucket(bucketKeys) + id2key := tx.Bucket(bucketIDs) + + // make a copy of rev2 that we can delete from, to see if + // any additions left in rev2 need to be added after all of + // rev is analyzed. + rev2cp := make(map[uint64]string) + for id, k := range rev2 { + rev2cp[id] = k + } + + // first we clean up any stale IDs from id2key. Then the fwd2 pass + // that follows will write to both key2id and id2key. + for id, key := range rev { + //vv("makeStringKeyChanges on rev2: id=%x -> key='%v'", id, key) + key2, ok := rev2[id] + if !ok { + changeIDs[id] = &BeforeAfterIDChange{IsDelete: true} + if verbose { + fmt.Printf("# %vkey-translation-delete-id: (id %x -> %v). Remaining for that key: ('%v' -> %x)\n", action, id, key, key, fwd2[key]) + } + if applyKeyRepairs { + u := u64tob(id) + err = id2key.Delete(u) + if err != nil { + return + } + } + continue + } + delete(rev2cp, id) + + if key2 != key { + u := u64tob(id) + k := []byte(key2) + changeIDs[id] = &BeforeAfterIDChange{ + BeforeString: key, + AfterString: key2, + } + if verbose { + fmt.Printf("# %vkey-translation-update-id: (id %x -> %v). fwd2 for that key: ('%v' -> %x)\n", action, id, key2, key2, fwd2[key2]) + } + if applyKeyRepairs { + err = id2key.Put(u, k) + if err != nil { + return + } + } + } + } + // anything leftover in rev2cp is stuff that is new, only + // in rev2 and not in rev. It needs to be added. + for id, key2 := range rev2cp { + u := u64tob(id) + k := []byte(key2) + changeIDs[id] = &BeforeAfterIDChange{ + IsAdd: true, + //BeforeString: left empty + AfterString: key2, + } + if verbose { + fmt.Printf("# %vkey-translation-add-id: (id %x -> %v). Fwd for that key: ('%v' -> %x)\n", action, id, key2, key2, fwd2[key2]) + } + if applyKeyRepairs { + err = id2key.Put(u, k) + if err != nil { + return + } + } + } + + // We assume here that fwd2 is a super-set of fwd. No string keys + // should be deleted in the repair. Confirm that. + for key, id := range fwd { + _, ok := fwd2[key] + if !ok { + panic(fmt.Sprintf("fwd2 is missing a string key from fwd. key='%v' -> id='%x'", key, id)) + } + } + + for key2, id2 := range fwd2 { + //vv("makeStringKeyChanges on fwd2, key2='%v', id2=%x", key2, id2) + isPrimary := false + if topo != nil { + primary := topo.GetPrimaryForColKeyTranslation(s.index, key2) + isPrimary = s.partitionID == primary + } + _ = isPrimary + id, ok := fwd[key2] + if !ok { + adds[key2] = id2 + + u2 := u64tob(id2) + k2 := []byte(key2) + if verbose { + fmt.Printf("# %vkey-translation-new-key: ('%v' -> %x) added: isPrimary: %v\n", action, key2, id2, isPrimary) + } + if applyKeyRepairs { + err = key2id.Put(k2, u2) + if err != nil { + return + } + err = id2key.Put(u2, k2) + if err != nil { + return + } + } + continue + } + if id != id2 { + changeKeys[key2] = &BeforeAfterKeyChange{ + BeforeID: id, + AfterID: id2, + } + if verbose { + fmt.Printf("# %vkey-translation-change-id: ('%v' -> %x) changes to ('%v' -> %x); isPrimary: %v\n", action, key2, id, key2, id2, isPrimary) + } + if applyKeyRepairs { + u2 := u64tob(id2) + k2 := []byte(key2) + + err = key2id.Put(k2, u2) + if err != nil { + return + } + err = id2key.Put(u2, k2) + if err != nil { + return + } + } + } + } + + if localTx { + err = tx.Commit() + } + return +} + +func (s *TranslateStore) DumpBolt(label string) { + + fmt.Printf("dumping bolt %v : path='%v'\n", label, s.Path) + + _ = s.KeyWalker(func(key string, col uint64) { + fmt.Printf("keyWalker: key '%v' -> col '%x'\n", key, col) + }) + _ = s.IDWalker(func(key string, col uint64) { + fmt.Printf("idWalker: id '%x' -> key '%v'\n", col, key) + }) + + fmt.Printf("DONE with dumping bolt %v; path='%v'\n", label, s.Path) + +} diff --git a/boltdb/translate_test.go b/boltdb/translate_test.go index 8c7a23c18..1e0803b1b 100644 --- a/boltdb/translate_test.go +++ b/boltdb/translate_test.go @@ -20,6 +20,7 @@ import ( "io/ioutil" "os" "reflect" + "strconv" "testing" "time" @@ -27,6 +28,8 @@ import ( "github.com/pilosa/pilosa/v2/boltdb" ) +//var vv = pilosa.VV + func TestTranslateStore_TranslateKey(t *testing.T) { s := MustOpenNewTranslateStore() defer MustCloseTranslateStore(s) @@ -216,6 +219,30 @@ func TestTranslateStore_TranslateIDs(t *testing.T) { } } +func TestTranslateStore_MaxID(t *testing.T) { + s := MustOpenNewTranslateStore() + defer MustCloseTranslateStore(s) + + // Generate a bunch of keys. + var lastk uint64 + for i := 0; i < 1026; i++ { + k, err := s.TranslateKey(strconv.Itoa(i), true) + if err != nil { + t.Fatalf("translating %d: %v", i, err) + } + lastk = k + } + + // Verify the max ID. + max, err := s.MaxID() + if err != nil { + t.Fatalf("checking max ID: %v", err) + } + if max != lastk { + t.Fatalf("last key is %d but max is %d", lastk, max) + } +} + func TestTranslateStore_EntryReader(t *testing.T) { t.Run("OK", func(t *testing.T) { s := MustOpenNewTranslateStore() @@ -508,8 +535,7 @@ func TestCryptoHashPerKey(t *testing.T) { } // done with setup - - sum, err := s.ComputeTranslatorSummary() + sum, err := s.ComputeTranslatorSummaryCols(0, pilosa.NewTopology(&pilosa.Jmphasher{}, pilosa.DefaultPartitionN, 1, nil)) if err != nil { panic(err) } @@ -533,3 +559,123 @@ func TestCryptoHashPerKey(t *testing.T) { } } + +func TestTranslateStore_RepairNonInvertibleStringKeyTranslation(t *testing.T) { + + const N = 6 + // before repair + var fwd [N]map[string]uint64 + var rev [N]map[uint64]string + + // after repair + var fwd2 [N]map[string]uint64 + var rev2 [N]map[uint64]string + + // case 0: forward is messed up (unlikely but check for it anyway, be sure we can repair) + // "key0" -> id 0 // correct. + // "key1" -> id 0 // wrong. after Repair, should see key1 -> 1 (0xec0002) + // + // id 0 -> "key0" // correct + // id 1 -> "key1" // correct + // + fwd[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00001} + rev[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + fwd2[0] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev2[0] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + + // case 1: reverse is messed up (we have seen this in the past) + // "key0" -> id 0 // correct + // "key1" -> id 1 // correct + // + // id 0 -> "key0" // correct. + // id 1 -> "key0" // wrong. after Repair, should see id 1 -> "key1" + // + fwd[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key0"} + fwd2[1] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev2[1] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + + // case 2: only present in reverse. + fwd[2] = map[string]uint64{} + rev[2] = map[uint64]string{0xec00001: "key0"} + fwd2[2] = map[string]uint64{"key0": 0xec00001} + rev2[2] = map[uint64]string{0xec00001: "key0"} + + // case 3: same thing. with camoflage. + fwd[3] = map[string]uint64{"key1": 0xec00002} + rev[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + fwd2[3] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev2[3] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + + // case 4: only present in forward. + + fwd[4] = map[string]uint64{"key0": 0xec00001} + rev[4] = map[uint64]string{} + fwd2[4] = map[string]uint64{"key0": 0xec00001} + rev2[4] = map[uint64]string{0xec00001: "key0"} + + // case 5: same thing. with camoflage. + fwd[5] = map[string]uint64{"key0": 0xec00001} + rev[5] = map[uint64]string{0xec00002: "key1"} + fwd2[5] = map[string]uint64{"key0": 0xec00001, "key1": 0xec00002} + rev2[5] = map[uint64]string{0xec00001: "key0", 0xec00002: "key1"} + + // case 6: we had an id, but b/c of the fix, that id is no longer used. + // now that id might still be used in the fragment for a column, + // and so we will need to remove that id/column from the fragment. + // encapsulated: "did it affect the state of the fields?" + + for i := 0; i < 5; i++ { + //println("i = ", i) + s := MustOpenNewTranslateStore() + defer MustCloseTranslateStore(s) + + if err := s.SetFwdRevMaps(nil, fwd[i], rev[i]); err != nil { + t.Fatal(err) + } + + if err := verifyState("setup", i, s, fwd[i], rev[i]); err != nil { + t.Fatal(err) + } + + var topo *pilosa.Topology + verbose := false + applyKeyRepairs := true + changed, err := s.RepairKeys(topo, verbose, applyKeyRepairs) + if err != nil { + t.Fatal(err) + } + if !changed { + t.Fatalf("expected changes!") + } + + if err := verifyState("afterRepair", i, s, fwd2[i], rev2[i]); err != nil { + t.Fatal(err) + } + } +} + +func verifyState(label string, i int, s *boltdb.TranslateStore, fwd map[string]uint64, rev map[uint64]string) error { + + // verify the setup + const writable = true + for key, expectID := range fwd { + id, err := s.TranslateKey(key, !writable) + if err != nil { + return err + } + if id != expectID { + return fmt.Errorf("fwd %v problem. i=%v, for key '%v', expected %x, observed %x", label, i, key, expectID, id) + } + } + for id, expectKey := range rev { + key, err := s.TranslateID(id) + if err != nil { + return err + } + if key != expectKey { + return fmt.Errorf("rev %v problem. i=%v, for id '%x', expected %v, observed %v", label, i, id, expectKey, key) + } + } + return nil +} diff --git a/cluster.go b/cluster.go index 6e5450cf4..23c86ac7b 100644 --- a/cluster.go +++ b/cluster.go @@ -62,7 +62,7 @@ const ( resizeJobActionAdd = "ADD" resizeJobActionRemove = "REMOVE" - defaultConfirmDownRetries = 120 + defaultConfirmDownRetries = 10 defaultConfirmDownSleep = 1 * time.Second ) @@ -208,6 +208,9 @@ type cluster struct { // nolint: maligned // The number of replicas a partition has. ReplicaN int + // Human-readable name of the cluster. + Name string + // Threshold for logging long-running queries // TODO(2.0) move this out of cluster. (why is it here??) longQueryTime time.Duration @@ -257,7 +260,7 @@ type cluster struct { // nolint: maligned // newCluster returns a new instance of Cluster with defaults. func newCluster() *cluster { return &cluster{ - Hasher: &jmphasher{}, + Hasher: &Jmphasher{}, partitionN: DefaultPartitionN, ReplicaN: 1, @@ -700,6 +703,12 @@ func (c *cluster) Nodes() []*Node { return ret } +func (c *cluster) AllNodeStates() map[string]string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.Topology.nodeStates +} + // removeNodeBasicSorted removes a node from the cluster, maintaining the sort // order. Returns true if the node was removed. unprotected. func (c *cluster) removeNodeBasicSorted(nodeID string) bool { @@ -969,12 +978,13 @@ func (c *cluster) translationNodes(to *cluster) (map[string][]*translationResize return m, nil } -// shardPartition returns the partition that a shard belongs to. -func (c *cluster) shardPartition(index string, shard uint64) int { - return shardPartition(index, shard, c.partitionN) +// shardPartition returns the shard-partition that a shard belongs to. +// NOTE: this is DIFFERENT from the key-partition +func (c *cluster) shardToShardPartition(index string, shard uint64) int { + return shardToShardPartition(index, shard, c.partitionN) } -func shardPartition(index string, shard uint64, partitionN int) int { +func shardToShardPartition(index string, shard uint64, partitionN int) int { var buf [8]byte binary.BigEndian.PutUint64(buf[:], shard) @@ -985,12 +995,13 @@ func shardPartition(index string, shard uint64, partitionN int) int { return int(h.Sum64() % uint64(partitionN)) } -// keyPartition returns the partition that a key belongs to. -func (c *cluster) keyPartition(index, key string) int { - return keyPartition(index, key, c.partitionN) +// keyPartition returns the key-partition that a key belongs to. +// NOTE: the key-partition is DIFFERENT from the shard-partition. +func (topo *Topology) KeyPartition(index, key string) int { + return keyToKeyPartition(index, key, topo.PartitionN) } -func keyPartition(index, key string, partitionN int) int { +func keyToKeyPartition(index, key string, partitionN int) int { // Hash the bytes and mod by partition count. h := fnv.New64a() _, _ = h.Write([]byte(index)) @@ -1000,7 +1011,7 @@ func keyPartition(index, key string, partitionN int) int { // idPartition returns the partition that an id belongs to. func (c *cluster) idPartition(index string, id uint64) int { - return shardPartition(index, id/ShardWidth, c.partitionN) + return shardToShardPartition(index, id/ShardWidth, c.partitionN) } // ShardNodes returns a list of nodes that own a fragment. Safe for concurrent use. @@ -1012,7 +1023,7 @@ func (c *cluster) ShardNodes(index string, shard uint64) []*Node { // shardNodes returns a list of nodes that own a fragment. unprotected func (c *cluster) shardNodes(index string, shard uint64) []*Node { - return c.partitionNodes(c.shardPartition(index, shard)) + return c.partitionNodes(c.shardToShardPartition(index, shard)) } // KeyNodes returns a list of nodes that own a fragment. Safe for concurrent use. @@ -1024,7 +1035,7 @@ func (c *cluster) KeyNodes(index, key string) []*Node { // keyNodes returns a list of nodes that own a key. unprotected func (c *cluster) keyNodes(index, key string) []*Node { - return c.partitionNodes(c.keyPartition(index, key)) + return c.partitionNodes(c.Topology.KeyPartition(index, key)) } // ownsShard returns true if a host owns a fragment. @@ -1068,8 +1079,14 @@ func (c *cluster) partitionNodes(partitionID int) []*Node { } // Determine primary owner node. - nodeIndex := c.Hasher.Hash(uint64(partitionID), nodeN) - + if c.Topology == nil { + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) + } + nodeIndex := c.Topology.PrimaryNodeIndex(partitionID) + if nodeIndex < 0 { + // no nodes anyway + return nil + } // Collect nodes around the ring. nodes := make([]*Node, 0, replicaN) for i := 0; i < replicaN; i++ { @@ -1100,11 +1117,66 @@ func (c *cluster) unprotectedPrimaryPartitionNode(partition int) *Node { return nil } +func (topo *Topology) IsPrimary(nodeID string, partitionID int) bool { + primary := topo.PrimaryNodeIndex(partitionID) + return nodeID == topo.nodeIDs[primary] +} + +func (topo *Topology) PrimaryNodeIndex(partitionID int) (nodeIndex int) { + n := len(topo.nodeIDs) + if n == 0 { + if topo.cluster != nil { + n = len(topo.cluster.nodes) + } + } + nodeIndex = topo.Hasher.Hash(uint64(partitionID), n) + return +} + +func (topo *Topology) GetNonPrimaryReplicas(partitionID int) (nonPrimaryReplicas []string) { + + primary := topo.PrimaryNodeIndex(partitionID) + nodeN := len(topo.nodeIDs) + + // Collect nodes around the ring. + for i := 1; i < nodeN; i++ { + nodeID := topo.nodeIDs[(primary+i)%nodeN] + if i < topo.ReplicaN { + nonPrimaryReplicas = append(nonPrimaryReplicas, nodeID) + } + } + return +} + +// the map replicaNodeIDs[nodeID] will have a true value for the primary nodeID, and false for others. +func (topo *Topology) GetReplicasForPrimary(primary int) (replicaNodeIDs, nonReplicas map[string]bool) { + if primary < 0 { + // no nodes anyway + return + } + replicaNodeIDs = make(map[string]bool) + nonReplicas = make(map[string]bool) + + nodeN := len(topo.nodeIDs) + + // Collect nodes around the ring. + for i := 0; i < nodeN; i++ { + nodeID := topo.nodeIDs[(primary+i)%nodeN] + if i < topo.ReplicaN { + // mark true if primary + replicaNodeIDs[nodeID] = (i == 0) + } else { + nonReplicas[nodeID] = false + } + } + return +} + // containsShards is like OwnsShards, but it includes replicas. func (c *cluster) containsShards(index string, availableShards *roaring.Bitmap, node *Node) []uint64 { var shards []uint64 _ = availableShards.ForEach(func(i uint64) error { - p := c.shardPartition(index, i) + p := c.shardToShardPartition(index, i) // Determine the nodes for partition. nodes := c.partitionNodes(p) for _, n := range nodes { @@ -1124,10 +1196,10 @@ type Hasher interface { } // jmphasher represents an implementation of jmphash. Implements Hasher. -type jmphasher struct{} +type Jmphasher struct{} // Hash returns the integer hash for the given key. -func (h *jmphasher) Hash(key uint64, n int) int { +func (h *Jmphasher) Hash(key uint64, n int) int { b, j := int64(-1), int64(0) for j < int64(n) { b = j @@ -1193,7 +1265,7 @@ func (c *cluster) waitForStarted() error { c.logger.Printf("%v wait for joining to complete", c.Node.ID) <-c.joining - c.logger.Printf("joining has completed") + c.logger.Printf("joining has completed. I am NodeID '%v'", c.Node.ID) } return nil } @@ -1844,6 +1916,8 @@ func (n nodeIDs) ContainsID(id string) bool { } // Topology represents the list of hosts in the cluster. +// Topology now encapsulates all knowledge needed to +// determine the primary node in the replication scheme. type Topology struct { mu sync.RWMutex nodeIDs []string @@ -1853,14 +1927,68 @@ type Topology struct { // nodeStates holds the state of each node according to // the coordinator. Used during startup and data load. nodeStates map[string]string + + // moved Hasher, PartitionN and ReplicaN + // from cluster for standalone use and comprehension: + + // Hashing algorithm used to assign partitions to nodes. + Hasher Hasher + // The number of partitions in the cluster. + PartitionN int + // The number of replicas a partition has. + ReplicaN int + + // can be nil + cluster *cluster } -func newTopology() *Topology { +// NewTopology creates a Topology. +// +// The arguments and members hasher, partitionN, and +// replicaN were refactored out of struct cluster +// to allow pilosa-fsck to load a Topology from +// backup and then compute primaries standalone -- without starting a cluster. +// As pilosa-fsck operates on all backups at once from +// a single cpu, starting a full cluster isn't possible. +// +// The hasher is the Hashing algorithm used to assign partitions to nodes. +// The cluster c should be provided if possible by pilosa code; +// the pilosa-fsck utility won't be able to provide it. +// +// For the cluster size N, the topology gives preference to +// len(t.nodeIDs) before falling back on len(c.nodes). +// +func NewTopology(hasher Hasher, partitionN int, replicaN int, c *cluster) *Topology { return &Topology{ + Hasher: hasher, + PartitionN: partitionN, + ReplicaN: replicaN, nodeStates: make(map[string]string), + cluster: c, } } +func (t *Topology) String() string { + return fmt.Sprintf(` +&pilosa.Topology{ + nodeIDs: %v, + clusterID: %v, + nodeStates: %v, + PartitionN: %v, + ReplicaN: %v, +} +`, + t.nodeIDs, + t.clusterID, + t.nodeStates, + t.PartitionN, + t.ReplicaN, + ) +} +func (t *Topology) GetNodeIDs() []string { + return t.nodeIDs +} + // ContainsID returns true if id matches one of the topology's IDs. func (t *Topology) ContainsID(id string) bool { t.mu.RLock() @@ -1924,7 +2052,7 @@ func (t *Topology) encode() *internal.Topology { func (c *cluster) loadTopology() error { buf, err := ioutil.ReadFile(filepath.Join(c.Path, ".topology")) if os.IsNotExist(err) { - c.Topology = newTopology() + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) return nil } else if err != nil { return errors.Wrap(err, "reading file") @@ -1934,7 +2062,7 @@ func (c *cluster) loadTopology() error { if err := proto.Unmarshal(buf, &pb); err != nil { return errors.Wrap(err, "unmarshalling") } - top, err := decodeTopology(&pb) + top, err := DecodeTopology(&pb, c.Hasher, c.partitionN, c.ReplicaN, c) if err != nil { return errors.Wrap(err, "decoding") } @@ -1945,7 +2073,6 @@ func (c *cluster) loadTopology() error { // saveTopology writes the current topology to disk. unprotected. func (c *cluster) saveTopology() error { - if err := os.MkdirAll(c.Path, 0777); err != nil { return errors.Wrap(err, "creating directory") } @@ -2452,6 +2579,27 @@ func (c *cluster) translateIndexKeys(ctx context.Context, indexName string, keys return ids, nil } +// The boltdb key translation stores are partitioned, designated by partitionIDs. These +// are shared between replicas, and one node is the primary for +// replication. So with 4 nodes and 3-way replication, each node has 3/4 of +// the translation stores on it. +func (topo *Topology) GetPrimaryForColKeyTranslation(index, key string) (primary int) { + partitionID := topo.KeyPartition(index, key) + return topo.PrimaryNodeIndex(partitionID) +} + +// should match cluster.go:1033 cluster.ownsShard(nodeID, index, shard) +// return Nodes(c.shardNodes(index, shard)).ContainsID(nodeID) +func (t *Topology) GetPrimaryForShardReplication(index string, shard uint64) int { + n := len(t.nodeIDs) + if n == 0 { + return -1 + } + partition := uint64(shardToShardPartition(index, shard, t.PartitionN)) + nodeIndex := t.Hasher.Hash(partition, n) + return nodeIndex +} + func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, keySet map[string]struct{}, writable bool) (map[string]uint64, error) { keyMap := make(map[string]uint64) @@ -2463,7 +2611,7 @@ func (c *cluster) translateIndexKeySet(ctx context.Context, indexName string, ke // Split keys by partition. keysByPartition := make(map[int][]string, c.partitionN) for key := range keySet { - partitionID := c.keyPartition(indexName, key) + partitionID := c.Topology.KeyPartition(indexName, key) keysByPartition[partitionID] = append(keysByPartition[partitionID], key) } @@ -2642,12 +2790,13 @@ func encodeTopology(topology *Topology) *internal.Topology { } } -func decodeTopology(topology *internal.Topology) (*Topology, error) { +// the cluster c is optional but give it if you have it. +func DecodeTopology(topology *internal.Topology, hasher Hasher, partitionN, replicaN int, c *cluster) (*Topology, error) { if topology == nil { return nil, nil } - t := newTopology() + t := NewTopology(hasher, partitionN, replicaN, c) t.clusterID = topology.ClusterID t.nodeIDs = topology.NodeIDs sort.Slice(t.nodeIDs, diff --git a/cluster_internal_test.go b/cluster_internal_test.go index 003e7f668..a9ec930be 100644 --- a/cluster_internal_test.go +++ b/cluster_internal_test.go @@ -381,7 +381,7 @@ func TestCluster_Partition(t *testing.T) { c := newCluster() c.partitionN = partitionN - partitionID := c.shardPartition(index, shard) + partitionID := c.shardToShardPartition(index, shard) if partitionID < 0 || partitionID >= partitionN { t.Errorf("partition out of range: shard=%d, p=%d, n=%d", shard, partitionID, partitionN) } @@ -411,7 +411,7 @@ func TestHasher(t *testing.T) { {0x0ddc0ffeebadf00d, []int{0, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 15, 15, 15, 15}}, } { for i, v := range tt.bucket { - hasher := &jmphasher{} + hasher := &Jmphasher{} if got := hasher.Hash(tt.key, i+1); got != v { t.Errorf("hash(%v,%v)=%v, want %v", tt.key, i+1, got, v) } @@ -1053,3 +1053,27 @@ func TestCluster_confirmNodeDownDown(t *testing.T) { t.Errorf("expected node to be down") } } + +func TestCluster_GetNonPrimaryReplicas(t *testing.T) { + + c := newCluster() + c.ReplicaN = 3 + topo := NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) + c.Topology = topo + nNodes := 4 + for i := 0; i < nNodes; i++ { + nodeID := fmt.Sprintf("node%d", i) + c.nodes = append(c.nodes, &Node{ + ID: nodeID, + URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), + }) + c.Topology.addID(nodeID) + } + + partitionID := 256 + nonPrimes := topo.GetNonPrimaryReplicas(partitionID) + m := len(nonPrimes) + if m != c.ReplicaN-1 { + t.Fatalf("expected 2 non primes, got %v", m) + } +} diff --git a/cmd/pilosa-chk/chk.go b/cmd/pilosa-chk/chk.go index f0d5d9258..353b33b3b 100644 --- a/cmd/pilosa-chk/chk.go +++ b/cmd/pilosa-chk/chk.go @@ -76,8 +76,10 @@ func main() { final := pilosa.NewAllTranslatorSummary() const verbose = true + const checkKeys = false + const applyKeyRepairs = false for _, idx := range holder.Indexes() { - asum, err := idx.ComputeTranslatorSummary(verbose) + asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs, nil, "fake-nodeID", 10) if err != nil { log.Fatal(err) } @@ -103,7 +105,7 @@ func main() { fmt.Printf("==============================\n") fmt.Printf("index: %v\n", idx.Name()) fmt.Printf("==============================\n") - idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog) + idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, nil, verbose) } } } diff --git a/cmd/pilosa-fsck/Makefile b/cmd/pilosa-fsck/Makefile new file mode 100644 index 000000000..1b1dcf14c --- /dev/null +++ b/cmd/pilosa-fsck/Makefile @@ -0,0 +1,36 @@ +.PHONY: install build release + +CLONE_URL=github.com/pilosa/pilosa +VERSION := $(shell git describe --tags 2> /dev/null || echo unknown) +LATTICE_COMMIT := $(shell git -C lattice rev-parse --short HEAD 2>/dev/null) +VARIANT = Molecula +VERSION_ID = $(VERSION)-$(GOOS)-$(GOARCH) +BRANCH := $(if $(TRAVIS_BRANCH),$(TRAVIS_BRANCH),$(if $(CIRCLE_BRANCH),$(CIRCLE_BRANCH),$(shell git rev-parse --abbrev-ref HEAD))) +BRANCH_ID := $(BRANCH)-$(GOOS)-$(GOARCH) +BUILD_TIME := $(shell date -u +%FT%T%z) +SHARD_WIDTH = 20 +COMMIT := $(shell git describe --exact-match >/dev/null 2>&1 || git rev-parse --short HEAD) +LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/pilosa/v2.BuildTime=$(BUILD_TIME) -X github.com/pilosa/pilosa/v2.Variant=$(VARIANT) -X github.com/pilosa/pilosa/v2.Commit=$(COMMIT) -X github.com/pilosa/pilosa/v2.LatticeCommit=$(LATTICE_COMMIT)" +GOOS = $(shell go env GOOS) + +# Install pilosa-fsck +install: + go install -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) + +# Compile pilosa-fsck +build: + go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) + +REL = release-pilosa-fsck.$(COMMIT).$(GOOS) + +release: + mkdir $(REL) + cd release-pilosa-fsck; tar cf - . |(cd ../$(REL); tar xf - ) + go build -tags='$(BUILD_TAGS)' -ldflags $(LDFLAGS) $(FLAGS) -o $(REL)/pilosa-fsck + tar cf - $(REL) | gzip > $(REL).tar.gz + rm -rf $(REL) + mv $(REL).tar.gz ../.. + +clean: + find . -name pilosa-fsck | xargs rm -f + rm -f release-pilosa-fsck*.tar.gz diff --git a/cmd/pilosa-fsck/fsck.go b/cmd/pilosa-fsck/fsck.go new file mode 100644 index 000000000..34ed071f8 --- /dev/null +++ b/cmd/pilosa-fsck/fsck.go @@ -0,0 +1,985 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/gogo/protobuf/proto" + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/boltdb" + "github.com/pilosa/pilosa/v2/internal" + "github.com/pilosa/pilosa/v2/server" + "github.com/pkg/errors" + "github.com/zeebo/blake3" +) + +// pilosa-fsck : +// an external customer tool (originally for Q2) to do 2 jobs: +// Given a set of cluster backups (and their .id and .topology files) +// mounted on the same file system, we can: +// 1) scan for fragment differences between the primary and its replicas (default); or +// 2) repair those differences by overwriting the replcas with the primary fragments (if -fix is given). +// +// pilosa-chk is deliberately NOT a part of pilosa so that it can run without +// forcing a customer to upgrade or downgrade their installed version. + +// FsckConfig configures the dumpcols() and/or read() runs. +type FsckConfig struct { + Fix bool // -fix + FixCol bool // -fixcol + + Colkeydump bool // -col + JustThisIndex string // -index + + // -col column key dump only options: + // Dir string + // PartitionID int + // ShowHeader bool + // ShowKey bool + // ShowID bool + + // not flags, just the Args() left after all other flags. Should be the list + // of pilosa (holder) directories for the cluster. + Dirs []string + + Verbose bool // -v + Quiet bool // -q + + // manual workaround for not having PilosaConfigPath, if really need be. + ReplicaN int // -replicas + PilosaConfigPath string // -config + + ParallelReaders int // -readers + + topo *pilosa.Topology +} + +// call DefineFlags before myflags.Parse() +func (cfg *FsckConfig) DefineFlags(fs *flag.FlagSet) { + fs.BoolVar(&cfg.Fix, "fix", false, "(warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. Implies -fixcol") + fs.BoolVar(&cfg.FixCol, "fixcol", false, "(warning: alters the backed-up node images on disk) repair string key translation tables. Skip repair of index data.") + //fs.BoolVar(&cfg.Verbose, "v", false, "be very verbose during analysis") + fs.BoolVar(&cfg.Quiet, "q", false, "be very quiet") + + fs.IntVar(&cfg.ReplicaN, "replicas", 0, "(required) manually entered replicaN; the number of replicas maintained in the cluster. Must be the same as the [cluster] 'replicas = R' entry in the pilosa.conf file for the cluster.") + + fs.IntVar(&cfg.ParallelReaders, "readers", 10, "how many parallel readers to use to scan at once. 0 means do everything possible in parallel. 1 means serialize everything through a single reader. Can be adjusted to control memory consumption.") + + fs.StringVar(&cfg.PilosaConfigPath, "config", "", "(required: -replicas or -config, with -config preferred) path to the pilosa.conf for the cluster (e.g. /etc/pilosa.conf)") + + fs.StringVar(&cfg.JustThisIndex, "index", "", "(optional) restrict to just this index. Otherwise we default to all indexes.") + + fs.Usage = func() { + fmt.Fprintf(os.Stderr, "pilosa-fsck version: %v\n\n", pilosa.VersionInfo()) + fmt.Fprintf(os.Stderr, `Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa + + -fix + (warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. + + -replicas R + (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is + the number of replicas maintained in the cluster. Must be the same as the + [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node. + + -index index_name + (optional) restrict to just this index. Otherwise we default to all indexes. + + -readers PR + how many parallel readers to use to scan at once. PR==0 means do everything + possible in parallel. PR==1 means serialize everything through a single reader. + Adjust PR to control memory consumption if needed. As a practical limit, setting + PR > 10000 will have no effect. (default is 10). + + -q + be very quiet during analysis and repair + +`) + fmt.Fprintf(os.Stderr, ` +Welcome to pilosa-fsck. This is a scan and repair +tool that is modeled after the classic unix file +system utility fsck. + +WARNING: DO NOT RUN ON A LIVE SYSTEM. + +The most important point to remember is that analysis +and repair must be done *offline*. + +Just as fsck must be run on an unmounted disk, +pilosa-fsck must be run on a backup. It must +not be run on the directories where a live Pilosa system +is serving queries. Instead, take a backup first. +A backup is a set of N Pilosa data directories that have been +copied from your live system. They must all +be visible and mounted on one filesystem together. + +pilosa-fsck can be run in scan-mode (without -fix), +or in repair-mode with -fix. The console output +supplies a log documenting the analysis +and showing what data changes would have been made. + +REQUIRED COMMAND LINE ARGUMENTS + +The paths to all the top-level Pilosa +data directories in a cluster must be given on the command +line. The -replicas R flag is also always required. It +must be correct for your cluser. Here R is the same as +the [cluster] stanza "replicas = R" line from your +pilosa.conf. + +Example: + +Suppose you are ready to run pilosa-fsck: +you have taken a backup of your four node Pilosa +cluster and stored it all on one filesystem with +all nodes visible and uncompressed. This +is a pre-requisite to running pilosa-fsck. +Let's suppose we have replication R = 3 set. +In this example, have stored our backed-up directories in + +/backup/molecula + +and the four node backups are in +subdirectories node1/ node2/ node3/ node4/ under this: + +/backup/molecula/node1/ +/backup/molecula/node1/.pilosa/.id +/backup/molecula/node1/.pilosa/.topology +/backup/molecula/node1/.pilosa/myindex + +/backup/molecula/node2/ +/backup/molecula/node2/.pilosa/.id +/backup/molecula/node2/.pilosa/.topology +/backup/molecula/node2/.pilosa/myindex + +/backup/molecula/node3/ +/backup/molecula/node3/.pilosa/.id +/backup/molecula/node3/.pilosa/.topology +/backup/molecula/node3/.pilosa/myindex + +/backup/molecula/node4/ +/backup/molecula/node4/.pilosa/.id +/backup/molecula/node4/.pilosa/.topology +/backup/molecula/node4/.pilosa/myindex + +NOTE: your .pilosa directories need not be named .pilosa. They can +be something else, such as when the -d flag to pilosa server was used. +The .id file, the .topology file, and the index directories must be +found directly underneath. + +Then a typical invocation to scan a cluster backup for issues: + +$ cd /backup/molecula/ +$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log + +A typical invocation to repair the replication in the same backup: + +$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log + +In both cases, the .id and .topology files must +be present in the backups. + +Without -fix, no modifications will be made to the backups. Only +by running with -fix will repairs be made. The user can safely +always run with -fix to repair only if needed. + +A zero error code will be returned to the shell if no repairs were needed. + +A zero error code will be also be returned to the shell if +repairs were needed and they were accomplished under -fix. + +A non-zero error code indicates that repairs were needed but +were not made. +`) + } +} + +// call c.ValidateConfig() after myflags.Parse() +func (c *FsckConfig) ValidateConfig() error { + if c.Fix { + c.FixCol = true + } + if c.ReplicaN == 0 && c.PilosaConfigPath == "" { + return fmt.Errorf("must supply -replicas with the replica count from your pilosa.conf (positive integer count)") + } + + if c.ReplicaN == 0 && c.PilosaConfigPath != "" { + + if !FileExists(c.PilosaConfigPath) { + return fmt.Errorf(" -config path '%v' does not exist", c.PilosaConfigPath) + } + by, err := ioutil.ReadFile(c.PilosaConfigPath) + if err != nil { + return fmt.Errorf("error: could not read the -config path '%v': '%v'", c.PilosaConfigPath, err) + } + srvcfg, err := server.ParseConfig(string(by)) + if err != nil { + //vv("warning: -config path '%v' problem, could not parse toml: '%v'", c.PilosaConfigPath, err) + + // fall back to manual parsing of config + lines := strings.Split(string(by), "\n") + clusterStart := -1 + for i, line := range lines { + if strings.Contains(line, `[cluster]`) { + clusterStart = i + } + if i > clusterStart { + if strings.Contains(line, "replicas") { + split := strings.Split(line, "=") + ns := strings.TrimSpace(split[1]) + n, err := strconv.Atoi(ns) + if err != nil { + return fmt.Errorf("error: could not parse the replicaN from line %v in -config path '%v' (%v): '%v'", i+1, c.PilosaConfigPath, line, err) + } + c.ReplicaN = n + } + } + } + } else { + c.ReplicaN = srvcfg.Cluster.ReplicaN + } + if c.ReplicaN == 0 { + return fmt.Errorf("error: -config path '%v' did not list the Replica count: cannot be 0. See the [cluster] section, the 'replicas = R' line.", c.PilosaConfigPath) + } + //vv("c.ReplicaN = %v", c.ReplicaN) + } + return nil +} + +var ProgramName = "pilosa-fsck" + +func main() { + + myflags := flag.NewFlagSet(ProgramName, flag.ContinueOnError) + cfg := &FsckConfig{} + cfg.DefineFlags(myflags) + cfg.Verbose = true + + err := myflags.Parse(os.Args[1:]) + if err != nil { + fmt.Fprintf(os.Stderr, "\n%v\n", err.Error()) + os.Exit(1) + } + err = cfg.ValidateConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err) + os.Exit(1) + } + dirs := myflags.Args() + nDir := len(dirs) + if nDir <= 0 && !cfg.Colkeydump { + fmt.Fprintf(os.Stderr, "error: %v command line arguments missing error: provide all of the top-level pilosa directories for the cluster as command line arguments.\n", ProgramName) + os.Exit(1) + } + + cmdline := strings.Join(os.Args, " ") + + // make sure all the dir are distinct + dup := make(map[string]bool) + for _, dir := range dirs { + if dup[dir] { + fmt.Fprintf(os.Stderr, "%v error: duplicate data directory '%v' given in command line '%v'. Each backup directory must be distinct.\n", ProgramName, dir, cmdline) + os.Exit(1) + } else { + dup[dir] = true + } + } + + fmt.Fprintf(os.Stdout, "#!/bin/bash\n\n# pilosa-fsck version: %v\n", pilosa.VersionInfo()) + cwd, err := os.Getwd() + if err != nil { + fmt.Fprintf(os.Stderr, "error: could not read current dir: '%v'\n", err) + os.Exit(1) + } + fmt.Fprintf(os.Stdout, "# cwd: %v\n", cwd) + fmt.Fprintf(os.Stdout, "# command line: %v\n", cmdline) + t0 := time.Now() + fmt.Fprintf(os.Stdout, "# started at %v\n\n", t0.Format(RFC3339MsecTz0)) + defer func() { + fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0)) + }() + cfg.Dirs = dirs + + fixNeeded, err := cfg.Run() + if err != nil { + fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0)) + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } + if fixNeeded && !cfg.Fix { + fmt.Fprintf(os.Stdout, "# finished at %v (elapsed %v)\n\n", time.Now().Format(RFC3339MsecTz0), time.Since(t0)) + fmt.Fprintf(os.Stderr, "# pilosa-fsck exiting with non-zero error code because a repair is needed, but -fix was not given.\n") + os.Exit(1) + } +} + +func (cfg *FsckConfig) Run() (fixNeeded bool, err error) { + + // if cfg.Colkeydump { + // cfg.dumpcols() + //} + + perNodeIndexMaps, clusterNodes, ats, err := cfg.read() + if err != nil { + return false, err + } + + if cfg.FixCol { + err := cfg.RepairTranslationStores(ats) + if err != nil { + return false, fmt.Errorf("error fixing key translation stores with cfg.RepairTranslationStores(): '%v'\n", err) + } + } + + //vv("perNodeIndexMaps='%#v', clusterNodes='%#v'", perNodeIndexMaps, clusterNodes) + + fixme, reports, err := cfg.analyze(clusterNodes, perNodeIndexMaps, ats) + if err != nil { + return false, fmt.Errorf("error in FsckConfig.analyze(): '%v'", err) + } + fixNeeded = ats.RepairNeeded || fixme + for _, report := range reports { + fmt.Printf("%v\n", report) + } + if len(reports) == 0 { + fmt.Fprintf(os.Stderr, "pilosa-fsck: no index found to analyze. cmdline was: %v\n", strings.Join(os.Args, " ")) + } + return +} + +var _ = (&FsckConfig{}).dumpAts + +func (cfg *FsckConfig) dumpAts(ats *pilosa.AllTranslatorSummary) { + fmt.Printf("# dumpAts: RepairNeeded=%v\n", ats.RepairNeeded) + for _, sum := range ats.Sums { + fmt.Printf("# sum = '%#v'\n", sum) + } + +} + +type group struct { + elem []*pilosa.TranslatorSummary + partitionID int +} + +func (g *group) String() (s string) { + for i, e := range g.elem { + s += fmt.Sprintf("partition %v, group elem [%v] out of %v: %v\n", g.partitionID, i, len(g.elem), e.String()) + } + return +} + +func indexesFromAts(ats *pilosa.AllTranslatorSummary) (indexes []string) { + indexMap := make(map[string]bool) + for _, sum := range ats.Sums { + if !indexMap[sum.Index] { + indexMap[sum.Index] = true + indexes = append(indexes, sum.Index) + } + } + sort.Strings(indexes) + return +} + +func (cfg *FsckConfig) RepairTranslationStores(ats *pilosa.AllTranslatorSummary) (err error) { + + verbose := cfg.Verbose + + // group by index first. then repair. + indexes := indexesFromAts(ats) + + for _, index := range indexes { + + if !cfg.DoingIndex(index) { + continue + } + + m := make(map[int]*group) + for _, sum := range ats.Sums { + + if !sum.IsColKey || sum.Index != index { + continue + } + grp := m[sum.PartitionID] + if grp == nil { + grp = &group{ + partitionID: sum.PartitionID, + } + m[sum.PartitionID] = grp + } + grp.elem = append(grp.elem, sum) + } + + for partitionID, group := range m { + _ = partitionID + prim := -1 + keyCount := 0 + for k, e := range group.elem { + if e.IsPrimary { + prim = k + } + keyCount += e.KeyCount + } + if prim == -1 { + panic(fmt.Sprintf("no primary found for group '%v'", group.String())) + } + + primary := group.elem[prim] + primaryChecksum := primary.Checksum + for _, e := range group.elem { + if e.IsPrimary { + continue + } + // is e a replica? not necessarily! have to check. + if !e.IsReplica { + //if verbose { + // since this will happen even on a fix point, where it is already empty, + // we don't report it again. + //fmt.Printf("# non-replica should have no data: creating an empty translation store here at '%v'\n", e.StorePath) + //} + err := os.RemoveAll(e.StorePath) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() os.RemoveAll(e.StorePath='%v')", e.StorePath)) + } + store, err := boltdb.OpenTranslateStore(e.StorePath, e.Index, e.Field, e.PartitionID, pilosa.DefaultPartitionN) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() create empty boldtdb: boltdb.OpenTranslateStore e.StorePath='%v'", e.StorePath)) + } + err = store.Close() + if err != nil { + return errors.Wrap(err, fmt.Sprintf("RepairTranslationStores() closing empty boltdb at path '%v'", e.StorePath)) + } + continue + } + // INVAR: e is a replica for this paritionID. + // Copy from primary if checksums are different. + if e.Checksum != primaryChecksum { + from := group.elem[prim].StorePath + dest := e.StorePath + if verbose { + fmt.Printf("# e.Checksum '%v' != primaryChecksum '%v': copying from primary translation store '%v' -> '%v'\n", e.Checksum, primaryChecksum, from, dest) + } + err := cp(from, dest) + if err != nil { + return fmt.Errorf("error: could not copy from primary '%v' to replica translation store '%v': '%v' ... try to keep going...\n", from, dest, err) + } + } + } + } + } + return nil +} + +/* +func (cfg *FsckConfig) dumpcols() { + + verbose := cfg.Verbose + quiet := cfg.Quiet + _, _ = verbose, quiet + + dir := cfg.Dir + index := cfg.Index + partitionID := cfg.PartitionID + showKey := cfg.ShowKey + showID := cfg.ShowID + + if !quiet { + fmt.Printf("# dumpcols: opening dir '%v'... this may take a few minutes...\n", dir) + } + holder := pilosa.NewHolder(dir, nil) + holder.OpenTranslateStore = boltdb.OpenTranslateStore + err := holder.Open() + if err != nil { + log.Fatal(err) + } + if cfg.ShowHeader { + fmt.Println("# columnKey columId") + } + id_key := make(map[uint64]string) + key_id := make(map[string]uint64) + for _, idx := range holder.Indexes() { + fmt.Printf("# Looking '%v'\n", idx.Name()) + if idx.Name() == index { + store := idx.TranslateStore(partitionID) + fmt.Printf("# Key By ID partitionID = %v\n", partitionID) + err := store.KeyWalker(func(key string, col uint64) { + key_id[key] = col + if showKey { + fmt.Printf("# '%v' %v shard: %v partition: %v\n", key, col, col/pilosa.ShardWidth, partitionID) + } + }) + panicOn(err) + } + } + for _, idx := range holder.Indexes() { + if idx.Name() == index { + store := idx.TranslateStore(partitionID) + //fmt.Printf("# ID ByKey\n") + err := store.IDWalker(func(key string, col uint64) { + id_key[col] = key + if showID { + fmt.Printf("# '%v' %v\n", key, col) + } + }) + panicOn(err) + } + } + fmt.Printf("# k: %d i: %d\n", len(key_id), len(id_key)) + fmt.Println("id_key") + for k, v := range id_key { + l, ok := key_id[v] + if ok { + if k != l { + fmt.Printf("# X: %v %v %v\n", k, l, v) + } + } else { + fmt.Printf("# key not in id %v\n", v) + } + } + fmt.Println("key_id") + for k, v := range key_id { + l, ok := id_key[v] + if ok { + if k != l { + fmt.Printf("# T: %v %v %v\n", k, l, v) + } + } else { + fmt.Printf("# id not in key %v\n", v) + } + } +} +*/ + +func (cfg *FsckConfig) read() (perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, clusterNodes []string, final *pilosa.AllTranslatorSummary, err error) { + + final = pilosa.NewAllTranslatorSummary() + + dirs := cfg.Dirs + for _, dir := range dirs { + idx2frag, nodeID, atsNode, err := cfg.readOneDir(dir) + if err != nil { + return nil, nil, nil, err + } + final.Append(atsNode) + clusterNodes = append(clusterNodes, nodeID) + perNodeIndexMaps = append(perNodeIndexMaps, idx2frag) + } + return +} + +func (cfg *FsckConfig) readOneDir(dir string) (idx2frag map[string]*pilosa.IndexFragmentSummary, nodeID string, atsNode *pilosa.AllTranslatorSummary, err error) { + + verbose := cfg.Verbose + quiet := cfg.Quiet + + if !quiet { + fmt.Printf("# opening dir '%v'... this may take a few minutes...\n\n", dir) + } + + jmphasher := &pilosa.Jmphasher{} + partitionN := pilosa.DefaultPartitionN + replicaN := cfg.ReplicaN + topo, err := loadTopology(dir, jmphasher, partitionN, replicaN) + if err != nil { + return nil, "", nil, err + } + cfg.topo = topo + //vv("topo = '%#v'", topo) + nodeIDs := topo.GetNodeIDs() + //vv("nodeIDs = '%#v'", nodeIDs) + nNodes := len(nodeIDs) + nDir := len(cfg.Dirs) + if nDir != nNodes { + return nil, "", nil, fmt.Errorf("command line had %v directories (%#v) but the .topology had %v nodes (%#v)", nDir, cfg.Dirs, nNodes, nodeIDs) + } + + holder := pilosa.NewHolder(dir, nil) + holder.OpenTranslateStore = boltdb.OpenTranslateStore + + nodeID, err = holder.LoadNodeID() + panicOn(err) + //vv("nodeID = '%v'", nodeID) + err = holder.Open() + + if err != nil { + log.Fatal(err) + } + + if !quiet { + fmt.Printf("\n# calculating hashes of row and column key translation maps on data from dir '%v'...\n", dir) + } + var indexes []*pilosa.Index + + const checkKeys = true + atsNode = pilosa.NewAllTranslatorSummary() + for _, idx := range holder.Indexes() { + + if !cfg.DoingIndex(idx.Name()) { + continue + } + + //vv("calling idx.ComputeTranslatorSummary(verbose, checkKeys=%v, cfg.FixCol='%v')", checkKeys, cfg.FixCol) + + asum, err := idx.ComputeTranslatorSummary(verbose, checkKeys, cfg.FixCol, topo, nodeID, cfg.ParallelReaders) + if err != nil { + log.Fatal(err) + } + atsNode.Append(asum) + indexes = append(indexes, idx) + } + atsNode.Sort() + + hasher := blake3.New() + if !quiet { + fmt.Printf("\n# summary of col/row translations in dir: %v:\n", dir) + } + for _, sum := range atsNode.Sums { + if !quiet { + fmt.Printf("# index: %v partitionID: %v blake3-%v keyCount: %v idCount: %v\n", sum.Index, sum.PartitionID, sum.Checksum, sum.KeyCount, sum.IDCount) + } + _, _ = hasher.Write([]byte(sum.Checksum)) + } + + var buf [16]byte + _, _ = hasher.Digest().Read(buf[0:]) + + if !quiet { + fmt.Printf("# all-checksum = blake3-%x\n", buf) + } + + // fragment analysis + + showBits := false + showOpsLog := false + idx2frag = make(map[string]*pilosa.IndexFragmentSummary) // on this node. + for _, idx := range indexes { + if verbose { + fmt.Printf("# ==============================\n") + fmt.Printf("# index: %v\n", idx.Name()) + fmt.Printf("# ==============================\n") + } + frgsum := idx.WriteFragmentChecksums(os.Stdout, showBits, showOpsLog, topo, verbose) + frgsum.Dir = dir + frgsum.NodeID = nodeID + idx2frag[idx.Name()] = frgsum + } + + _ = holder.Close() + + //vv("idx2frag = '%v'", idx2frag) // tons of output. see 1234.out.full for examaple. + + return +} + +func (cfg *FsckConfig) DoingIndex(index string) bool { + if cfg.JustThisIndex == "" { + // scan all indexes + return true + } + if index == cfg.JustThisIndex { + // scan just this one + return true + } + return false +} + +// from cluster.go:1924 +func loadTopology(holderDir string, hasher pilosa.Hasher, partitionN, replicaN int) (*pilosa.Topology, error) { + + buf, err := ioutil.ReadFile(filepath.Join(holderDir, ".topology")) + if err != nil { + return nil, err + } + + var pb internal.Topology + err = proto.Unmarshal(buf, &pb) + if err != nil { + return nil, err + } + + return pilosa.DecodeTopology(&pb, hasher, partitionN, replicaN, nil) +} + +func (cfg *FsckConfig) analyze(clusterNodes []string, perNodeIndexMaps []map[string]*pilosa.IndexFragmentSummary, ats *pilosa.AllTranslatorSummary) (fixNeeded bool, reports []string, err error) { + + verbose := cfg.Verbose + quiet := cfg.Quiet + _, _ = verbose, quiet + + allIndex := make(map[string]bool) + for _, mp := range perNodeIndexMaps { + for index := range mp { + allIndex[index] = true + } + } + if !quiet { + vv("allIndex = '%#v'", allIndex) + } + for index := range allIndex { + if !quiet { + vv("on index '%v'", index) + } + nodes2fragsum := make(map[string]*pilosa.IndexFragmentSummary) + for _, mp := range perNodeIndexMaps { + sum := mp[index] + if sum == nil { + continue + } + nodes2fragsum[sum.NodeID] = sum + } + fixme, report, err := cfg.analyzeThisIndex(index, nodes2fragsum, ats) + if err != nil { + return false, reports, fmt.Errorf("error in analyze of index '%v': '%v'", index, err) + } + fixNeeded = fixNeeded || fixme + reports = append(reports, report) + } + return fixNeeded, reports, nil +} + +func (cfg *FsckConfig) analyzeThisIndex( + index string, + nodes2fragsum map[string]*pilosa.IndexFragmentSummary, + ats *pilosa.AllTranslatorSummary, +) (fixNeeded bool, report string, err error) { + + verbose := cfg.Verbose + quiet := cfg.Quiet + _, _ = verbose, quiet + + var removedBytes int64 + var copiedBytes int64 + var changedFiles int64 + var totalFiles int64 + var overwrittenBytes int64 + var totalBytes int64 + + if !quiet { + vv("top of analyzeThisIndex(index='%v'); len of nodes2fragsum = %v; nodes2fragsum='%#v'", + index, len(nodes2fragsum), nodes2fragsum) + } + + for node, sum := range nodes2fragsum { + if !quiet { + fmt.Printf("# on node '%v'\n", node) + } + // do they disagree on who is the primary? + // for each fragment, do they disagree on the checksum? + + // Q: which nodes are supposed to have data, and which + // nodes are not supposed to have data? + + // loopFragSum: + for relpath, fragsum := range sum.RelPath2fsum { + fragsum.NodeID = node + totalFiles++ + //vv("checking %v on node %v", relpath, node) + + replicas, nonReplicas := cfg.topo.GetReplicasForPrimary(fragsum.Primary) + _, _ = replicas, nonReplicas + //vv("replicas = '%#v'", replicas) + //vv("nonReplicas = '%#v'", nonReplicas) + + err := cfg.verifyReplicasAvailable(replicas, nonReplicas, nodes2fragsum, fragsum) + if err != nil { + return fixNeeded, "", err + } + + // find the primary's checksum + primaryChecksum := "" + var primaryFragSum *pilosa.FragSum + for node, isPrimary := range replicas { + if isPrimary { + primarySum := nodes2fragsum[node] + primaryFragSum = primarySum.RelPath2fsum[relpath] + if primaryFragSum == nil { + + // This seems clear indication that we have the topology wrong. + // When the topology is right, there are NO errors of this kind. + // + msg := fmt.Sprintf("# ugh. BAD. Stopping because any fix will be wrong. We see wrong -replica %v param, OR the .id files are mis-assigned with respect to the topology file. Could not find primary FragSum for relpath = '%v'. replicas = '%#v', nonReplicas = '%#v'\n", cfg.ReplicaN, relpath, replicas, nonReplicas) + vv(msg) + fmt.Fprintf(os.Stderr, "%v\n", msg) + panic(msg) // stop. the fixes are going to be wrong. + } else { + primaryChecksum = primaryFragSum.Checksum + primaryFragSum.NodeID = node + primaryFragSum.ScanDone = true + } + break + } + } + if primaryChecksum == "" { + return fixNeeded, "", fmt.Errorf("could not find primary replica??? replicas='%#v', nodes2fragsum='%v'; for fragsum='%#v'", replicas, nodes2fragsum, fragsum) + } + + // is this a non-replica? + _, isNon := nonReplicas[fragsum.NodeID] + if isNon { + removedBytes += FileSize(fragsum.AbsPath) + changedFiles++ + + //vv("yes, is nonReplica: fragsum.NodeID='%v'", fragsum.NodeID) + if !quiet { + fmt.Printf("rm %v #### REPAIR REMOVE data from non-replica at node '%v' (fragsum='%#v') vs. primary (%#v)\n\n", fragsum.AbsPath, node, fragsum, primaryFragSum) + } + if cfg.Fix { + err := os.Remove(fragsum.AbsPath) + if err != nil { + return fixNeeded, "", fmt.Errorf("error removing non-replica extra fragment '%v': '%v'", fragsum.AbsPath, err) + } + } + } else { + presz := FileSize(fragsum.AbsPath) + totalBytes += presz + + checksum := fragsum.Checksum + if checksum != primaryChecksum { + copiedBytes += FileSize(primaryFragSum.AbsPath) + changedFiles++ + overwrittenBytes += presz + + if !quiet { + fmt.Printf("cp %v %v #### REPAIR OVERWRITE replica at node '%v' (%#v) from primary '%v' (%#v)\n", primaryFragSum.AbsPath, fragsum.AbsPath, node, fragsum, primaryFragSum.NodeID, primaryFragSum) + } + if cfg.Fix { + err := cp(primaryFragSum.AbsPath, fragsum.AbsPath) + if err != nil { + return fixNeeded, "", fmt.Errorf("error copying from '%v' to '%v': '%v'", + primaryFragSum.AbsPath, fragsum.AbsPath, err) + } + } + } + } + fragsum.ScanDone = true + } + } + nDir := len(nodes2fragsum) + + keyCount, idCount := cfg.getKeyIDCounts(index, ats) + + fixNeeded = changedFiles > 0 || ats.RepairNeeded + var actionTaken string + var wouldBe string + if cfg.Fix || cfg.FixCol { + if fixNeeded { + actionTaken = "*REPAIRS WERE MADE TO THE BACKUPS*" + wouldBe = "sync repairs made:" + } else { + wouldBe = "" + actionTaken = "NO REPAIR NEEDED." + } + } else { + if fixNeeded { + wouldBe = "sync actions that would be taken under -fix:" + actionTaken = "*REPAIRS NEEDED BUT WERE NOT APPLIED* ; pilosa-fsck -fix was omitted." + } else { + wouldBe = "" + actionTaken = "NO REPAIR NEEDED." + } + } + var fragUpdate string + if changedFiles > 0 { + fragUpdate = fmt.Sprintf(` +# %v +# copied bytes: %v +# file bytes overwritten: %v +# new bytes added: %v +# new bytes is %0.01f%% of %v total bytes +# removed %v bytes from non-replicas +# changed file count %v (%0.01f%%; total files=%v) +# +`, wouldBe, humanize.Comma(copiedBytes), humanize.Comma(overwrittenBytes), humanize.Comma(copiedBytes-overwrittenBytes), 100*float64(copiedBytes-overwrittenBytes)/float64(totalBytes), humanize.Comma(totalBytes), humanize.Comma(removedBytes), changedFiles, 100*float64(changedFiles)/float64(totalFiles), humanize.Comma(totalFiles)) + } + + report = fmt.Sprintf(` +# ======================================================== +# pilosa-fsck final report +# +# run with -fix: %v +# +# index examined: '%v' +# +# nodes examined: %v +# -replicas %v replication factor used +# +# feature data examined: %v bytes +# feature files examined: %v files +# +# key-translation-stores examined: %v +# key-count: %v over all replicas +# id-count: %v over all replicas +# +# %v +# %v +# ======================================================== +`, + cfg.Fix, index, nDir, cfg.ReplicaN, humanize.Comma(totalBytes), humanize.Comma(totalFiles), humanize.Comma(int64(nDir*pilosa.DefaultPartitionN)), humanize.Comma(int64(keyCount)), humanize.Comma(int64(idCount)), actionTaken, fragUpdate) + return +} + +func (cfg *FsckConfig) verifyReplicasAvailable(replicas, nonReplicas map[string]bool, nodes2fragsum map[string]*pilosa.IndexFragmentSummary, fragsum *pilosa.FragSum) error { + for node := range replicas { + if nodes2fragsum[node] == nil { + return fmt.Errorf("error: node '%v' needed for a replica set was not availabe. Did you give ALL the directories for your cluster on the command line at once? In nodes2fragsum '%#v' (replicas: '%#v'; non-replicas '%#v') for fragsum '%v'", node, nodes2fragsum, replicas, nonReplicas, fragsum) + } + } + return nil +} + +func cp(fromPath, toPath string) (err error) { + tmpTo := toPath + ".fsck.tmp" + toFd, err := os.Create(tmpTo) + if err != nil { + return err + } + defer toFd.Close() + fromFd, err := os.Open(fromPath) + if err != nil { + return err + } + defer fromFd.Close() + + _, err = io.Copy(toFd, fromFd) + if err != nil { + return err + } + err = toFd.Close() + if err != nil { + return err + } + return os.Rename(tmpTo, toPath) +} + +func (cfg *FsckConfig) getKeyIDCounts(index string, ats *pilosa.AllTranslatorSummary) (keyCount, idCount int) { + for _, sum := range ats.Sums { + if sum.Index == index { + keyCount += sum.KeyCount + idCount += sum.IDCount + } + } + return +} diff --git a/cmd/pilosa-fsck/fsck_test.go b/cmd/pilosa-fsck/fsck_test.go new file mode 100644 index 000000000..41e0d7a8c --- /dev/null +++ b/cmd/pilosa-fsck/fsck_test.go @@ -0,0 +1,445 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "io/ioutil" + "reflect" + "strconv" + "testing" + "time" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/boltdb" + "github.com/pilosa/pilosa/v2/hash" + "github.com/pilosa/pilosa/v2/http" + "github.com/pilosa/pilosa/v2/server" + "github.com/pilosa/pilosa/v2/test" +) + +func Test_Repair(t *testing.T) { + + // a) setup 1 primary + 3 replicas of disagree-ing cluster dirs. + + nNodes := 4 + nReplicas := 3 + + name := t.Name() + var nodeid []string + for i := 0; i < nNodes; i++ { + // work around a bug in the test.MustRunCluster that corrupts + // the .topology file if we only join name with one "_" underscore. + nodeid = append(nodeid, name+"__"+strconv.Itoa(i)) + } + + c := test.MustRunCluster(t, nNodes, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[0]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[1]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[2]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[3]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + ) + // note: do not defer c.Close() here. We manually close below. + + var nodes []*test.Command + var dirs []string + for i := 0; i < nNodes; i++ { + nd := c.GetNode(i) + nodes = append(nodes, nd) + dirs = append(dirs, nd.Server.Holder().Path()) + } + + ctx := context.Background() + + index := []string{"rick", "morty"} + fieldName := []string{"f", "flying_car"} + idx := make([]*pilosa.Index, len(index)) + field := make([]*pilosa.Field, len(index)) + var err error + + for i := range index { + + idx[i], err = nodes[0].API.CreateIndex(ctx, index[i], pilosa.IndexOptions{Keys: true, TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + if idx[i].CreatedAt() == 0 { + t.Fatal("index createdAt is empty") + } + + field[i], err = nodes[0].API.CreateField(ctx, index[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + if field[i].CreatedAt() == 0 { + t.Fatal("field createdAt is empty") + } + } + + rowID := uint64(1) + timestamp := int64(0) + + for i := range index { + + // Generate some keyed records. + rowIDs := []uint64{} + timestamps := []int64{} + N := 10 + for j := 1; j <= N; j++ { + rowIDs = append(rowIDs, rowID) + timestamps = append(timestamps, timestamp) + } + + var colKeys []string + switch i { + case 0: + // Keys are sharded so ordering is not guaranteed. + colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} + colKeys = colKeys[:N] + case 1: + colKeys = []string{"col11", "col12"} + N = len(colKeys) + rowIDs = rowIDs[:N] + timestamps = timestamps[:N] + } + + // Import data with keys to the coordinator (node0) and verify that it gets + // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) + req := &pilosa.ImportRequest{ + Index: index[i], + IndexCreatedAt: idx[i].CreatedAt(), + Field: fieldName[i], + FieldCreatedAt: field[i].CreatedAt(), + + // even though this says Shard: 0, that won't matter. The column keys + // get hashed and that decides the actual shard. + Shard: 0, + RowIDs: rowIDs, + ColumnKeys: colKeys, + Timestamps: timestamps, + } + + qcx := nodes[0].API.Txf().NewQcx() + + if err := nodes[0].API.Import(ctx, qcx, req); err != nil { + t.Fatal(err) + } + panicOn(qcx.Finish()) + //qcx.Reset() + + pql := fmt.Sprintf("Row(%s=%d)", fieldName[i], rowID) + + // Query node0. + if res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil { + t.Fatal(err) + } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { + t.Fatalf("expected colKeys='%#v'; observed column keys: %#v", colKeys, keys) + } + + // Query node1. + if err := test.RetryUntil(5*time.Second, func() error { + if res, err := nodes[1].API.Query(ctx, &pilosa.QueryRequest{Index: index[i], Query: pql}); err != nil { + return err + } else if keys := res.Results[0].(*pilosa.Row).Keys; !reflect.DeepEqual(keys, colKeys) { + return fmt.Errorf("unexpected column keys: %#v", keys) + } + return nil + }); err != nil { + t.Fatal(err) + } + } + // end of setup. + + // partitionID in use: 6, 31, 57, 133, 185, 235 + targetPartition := 31 // which partitionID we mess with. + targetNode := nodes[0] // this is the first replica. + targetIndex := index[0] + // 0 first replica + // 1 second replica + // 2 -- not a replica + // 3 primary + + cfg := &FsckConfig{ + Fix: false, + FixCol: false, + Quiet: true, + //Verbose: true, + ReplicaN: nReplicas, + Dirs: dirs, + ParallelReaders: 5, + } + panicOn(cfg.ValidateConfig()) + + // for this test, mess up a replica that is not the primary. + + h := targetNode.API.Holder() + idx[0] = h.Index(index[0]) + store := idx[0].TranslateStore(targetPartition) + fwd, rev := getFwdRev(store, targetPartition) + //vv("targetPartition=%v, store.PartitionID=%v, before corruption, fwd='%#v', rev='%#v'", targetPartition, store.PartitionID, fwd, rev) + + // # fsck_test.go:288 2020-10-01T13:39:57.718995-05:00 partition 31, key 'col5' -> db00001 + presz := len(rev) + delete(rev, fwd["col5"]) + postsz := len(rev) + + if postsz == presz { + panic("did not delete any key!") + } + + bolt := store.(*boltdb.TranslateStore) + //vv("pre corruption, bolt = '%v'", fileChecksum(bolt.Path)) + //bolt.DumpBolt("pre-corruption") + + if err := bolt.SetFwdRevMaps(nil, fwd, rev); err != nil { + t.Fatal(err) + } + //vv("post corruption, bolt = '%v'", fileChecksum(bolt.Path)) + //bolt.DumpBolt("post-corruption") + + //fwd3, rev3 := getFwdRev(store, targetPartition) + //vv("after corruption, fwd='%#v', rev='%#v'", fwd3, rev3) + + targetIndex1 := "morty" + targetPartition1 := 226 // for "col11" + // # fsck_test.go:248 2020-10-06T20:24:33.755576-05:00 on k=47, idx[1]: targetPartition=47, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col12":0xcf00001}', rev1='map[uint64]string{0xcf00001:"col12"}' + //# fsck_test.go:248 2020-10-06T20:24:35.608568-05:00 on k=226, idx[1]: targetPartition=226, store.PartitionID=0x4abe160, before corruption, fwd1='map[string]uint64{"col11":0xcc00001}', rev1='map[uint64]string{0xcc00001:"col11"}' + idx[1] = h.Index(index[1]) + store1 := idx[1].TranslateStore(targetPartition1) + fwd1, rev1 := getFwdRev(store1, targetPartition1) + //vv("on k=%v, idx[1]: targetPartition=%v, store.PartitionID=%v, before corruption, fwd1='%#v', rev1='%#v'", k, targetPartition1, store.PartitionID, fwd1, rev1) + + presz1 := len(rev1) + delete(rev1, fwd1["col11"]) + postsz1 := len(rev1) + + if postsz1 == presz1 { + panic("did not delete any key!") + } + bolt1 := store1.(*boltdb.TranslateStore) + if err := bolt1.SetFwdRevMaps(nil, fwd1, rev1); err != nil { + t.Fatal(err) + } + + // done corrupting. + for _, nd := range nodes { + nd.Command.Close() + } + //panicOn(bolt.Open()) + //bolt.DumpBolt("post-corruption, after Close. bolt:") + //bolt.Close() + + //chksums := getChecksums(dirs, cfg, targetPartition) + //vv("post corruption, pre repair chksums = '%#v'", chksums) + + // first we check that the corruption can be detected + // by our test with the checksums. + + chk, err := check(dirs, cfg, targetIndex, targetPartition) + _ = chk + //vv("pre-fix, chk='%v'; err='%v'", chk, err) + + if err == nil { + panic("expected to see checksums not match! but no corruption detected.") + } + + chk1, err := check(dirs, cfg, targetIndex1, targetPartition1) + _ = chk1 + //vv("pre-fix, chk1='%v'; err='%v'", chk1, err) + + if err == nil { + panic("expected to see checksums not match! but no corruption detected.") + } + + // b) running in reporting mode only should report that a fix is needed. + fixNeeded, err := cfg.Run() + panicOn(err) + if !fixNeeded { + panic("fix should be needed now, before repair") + } + + // c) run the fix. + cfg.Fix = true + cfg.FixCol = true + + fixNeeded, err = cfg.Run() + panicOn(err) + if !fixNeeded { + panic("fix should be marked needed if repair was made") + } + + // d) check that the replicas all look like the primary. + + //chksums = getChecksums(dirs, cfg, targetPartition) + //vv("after repair chksums = '%#v'", chksums) + + chk, err = check(dirs, cfg, targetIndex, targetPartition) + _ = chk + //vv("chk = '%v' after repair; err='%v'", chk, err) + panicOn(err) + + chk1, err = check(dirs, cfg, targetIndex1, targetPartition1) + _ = chk1 + //vv("chk = '%v' after repair; err='%v'", chk, err) + panicOn(err) + + // e) run again, should see no fix needed. + fixNeeded, err = cfg.Run() + panicOn(err) + if fixNeeded { + panic("should see no fix needed after the prior repair") + } +} + +func getFwdRev(store pilosa.TranslateStore, partitionID int) (fwd map[string]uint64, rev map[uint64]string) { + fwd = make(map[string]uint64) + rev = make(map[uint64]string) + _ = store.KeyWalker(func(key string, col uint64) { + //vv("partition %v, key '%v' -> %x", partitionID, key, col) + fwd[key] = col + }) + _ = store.IDWalker(func(key string, col uint64) { + //vv("partition %v, id %x -> '%v'", partitionID, col, key) + rev[col] = key + }) + return +} + +func check(dirs []string, cfg *FsckConfig, targetIndex string, targetPartition int) (chksum string, err error) { + //vv("top of check, dirs = '%#v', targetIndex='%v', targetPartition='%v'", dirs, targetIndex, targetPartition) + //defer vv("returning from check()") + + firstChecksum := "" + firstDir := "" + firstStorePath := "" + quiet := cfg.Quiet + defer func() { + cfg.Quiet = quiet + }() + cfg.Quiet = true + for i := range dirs { + dir := dirs[i] + _, _, ats, err := cfg.readOneDir(dir) + panicOn(err) + indexes := indexesFromAts(ats) + //vv("indexes = '%#v'", indexes) + + for _, index := range indexes { + + if index != targetIndex { + continue + } + for _, s := range ats.Sums { + //vv(" s= '%#v'", s) + if s.Index != index { + //vv("skipping s.Index '%v' != index '%v'", s.Index, index) + continue + } + if s.PartitionID != targetPartition { + continue + } + //vv("accepting s.PartitionID(%v) == targetPartition(%v); s.Index '%v'; "+ + //"index '%v'; s.IsPrimary=%v, s.IsReplica=%v, s='%#v'; s.Checksum='%v', firstChecksum='%v'", + //s.PartitionID, targetPartition, s.Index, index, + //s.IsPrimary, s.IsReplica, s, s.Checksum, firstChecksum) + + if s.IsPrimary || s.IsReplica { + chksum := s.Checksum + if firstChecksum == "" { + + firstChecksum = chksum + firstDir = dir + firstStorePath = s.StorePath + + } else { + //vv("targetIndex = '%v'; firstChecksum='%v', chksum='%v'", targetIndex, firstChecksum, chksum) + + if chksum != firstChecksum { + return chksum, fmt.Errorf("bolt chksum on node %v '%v' disagrees with '%v' on '%v'; index='%v'; s.StorePath = '%v'; firstStorePath='%v'", dir, chksum, firstChecksum, firstDir, index, s.StorePath, firstStorePath) + } + } + } + } + } + } + return firstChecksum, nil +} + +var _ = getChecksums + +func getChecksums(dirs []string, cfg *FsckConfig, targetPartition int) (chksum []string) { + + for i := range dirs { + dir := dirs[i] + _, _, ats, err := cfg.readOneDir(dir) + panicOn(err) + + for _, s := range ats.Sums { + if s.PartitionID != targetPartition { + continue + } + chksum = append(chksum, s.Checksum) + } + } + return +} + +/* on shardwidth 20 +# fsck_test.go:211 2020-09-30T17:19:05.823278-05:00 partition 6, key 'col2' -> dc00001 +# fsck_test.go:214 2020-09-30T17:19:05.823309-05:00 partition 6, id dc00001 -> 'col2' +# fsck_test.go:211 2020-09-30T17:19:05.823430-05:00 partition 31, key 'col5' -> db00001 +# fsck_test.go:214 2020-09-30T17:19:05.823447-05:00 partition 31, id db00001 -> 'col5' +# fsck_test.go:211 2020-09-30T17:19:05.823970-05:00 partition 57, key 'col10' -> 5d00001 +# fsck_test.go:214 2020-09-30T17:19:05.823998-05:00 partition 57, id 5d00001 -> 'col10' +# fsck_test.go:211 2020-09-30T17:19:05.827007-05:00 partition 133, key 'col7' -> d900001 +# fsck_test.go:214 2020-09-30T17:19:05.827071-05:00 partition 133, id d900001 -> 'col7' +# fsck_test.go:211 2020-09-30T17:19:05.827549-05:00 partition 185, key 'col3' -> dd00001 +# fsck_test.go:214 2020-09-30T17:19:05.827573-05:00 partition 185, id dd00001 -> 'col3' +# fsck_test.go:211 2020-09-30T17:19:05.827792-05:00 partition 235, key 'col9' -> d700001 +# fsck_test.go:214 2020-09-30T17:19:05.827809-05:00 partition 235, id d700001 -> 'col9' +*/ + +var _ = fileChecksum + +func fileChecksum(path string) string { + by, err := ioutil.ReadFile(path) + panicOn(err) + return hash.Blake3sum16(by) +} diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/.gitignore b/cmd/pilosa-fsck/release-pilosa-fsck/.gitignore new file mode 100644 index 000000000..a08586f1c --- /dev/null +++ b/cmd/pilosa-fsck/release-pilosa-fsck/.gitignore @@ -0,0 +1 @@ +pilosa-fsck diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md b/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md new file mode 100644 index 000000000..598896c8d --- /dev/null +++ b/cmd/pilosa-fsck/release-pilosa-fsck/DESIGN.md @@ -0,0 +1,252 @@ +Design for pilosa-fsck +====================== + +Problem Background +------------------ + +Molecula Pilosa provides replication for fault-tolerance within a Pilosa cluster. + +Three kinds of data are replicated: Roaring bitmap data, Column-Key translation data, +and Row-Key data are replicated. Only the first two, Roaring data and Column-Key +data are relevant here. Broadly, the Roaring bitmap data +forms the central features -- the bits -- of a large, sparse bitmap matrix. +The Column-Keys are the labels for the columns at the top margin of this matrix. + +For speed, the Roaring bitmap data is stored separately from the +Key data. The Roaring data is stored in sharded files +within a directory heirarchy under PILOSA-DATA-DIR/index_name/field_name/... +The Key translation data is stored in sharded BoltDB databases within +the PILOSA-DATA-DIR/index_name/_key directory. + +The current approach to Roaring file replication involves an +eventually consistent mechanism that uses an Anti-Entropy agent to +fix partial or incomplete replication from the primary shard to all +replica shards. + +Unfortunately, the Anti-Entropy agent approach has proved inadequate on two +fronts. First, it does not provide for immediately consistent reads in the +event that the primary is lost. Second, the Anti-Entropy agent itself experienced +out-of-memory issues that have yet to be resolved. + +Therefore, work is now underway to replace this replication +approach with a more consistent design. + +However, in the meantime, for our customers in production with Molecula +Pilosa, we wish to provide a means to re-establish correct replication. +Thus even in the event of a node failure followed by a read from a replica, the +returned read will be correct. + +The pilosa-fsck tool can therefore be seen as a temporary, stop-gap +measure to address immediate issues while the cluster replication +mechanism is replaced. + +The second factor motivating the creation of pilosa-fsck was the discovery +of a bug in the Key-translation process. Unfortunately this was a hard +to reproduce bug. It happened only on the customer's premises, +and only after running the system for a long time, with a +large amount of data, and with various eccentric node failures +and recoveries. + +However, we were able to reproduce a plausible explanation. +Non-primary replicas were creating keys when they should have been +forwarding the request to the primary. Correcting this bug is impetus +for the v2.1.4 release of Molecula Pilosa. + +A fine point here: since we were not able to precisely reproduce the customer's +issue in the development environment, we cannot guarantee with 100% +certainty that we have actually addressed the bug that the customer +was seeing. + +Therefore we also desired an additional insurance +policy. We wished to be able to empower customers to proactively discover any +future Key-translation issues that happen in their on-premise systems. + +To do this, we proposed providing select customers with the pilosa-fsck +tool which can analyze their offline backups for issues. + +Optionally, these issues can also be repaired in-place in the +offline backup on which pilosa-fsck is run. + +The -fix flag repairs both kinds of replication issues. + +Solution Approach: mechanism of action +-------------------------------------- + +The pilosa-fsck is run offline on a full set of backups taken from +all nodes in a Pilosa cluster. It runs on a single computer that +must be separate from the production or staging Pilosa environments. + +When run, pilosa-fsck analyzes the differences between the +primary and its replicas. Both the Roaring +files and the Key translation databases are analyzed. +The computer running pilosa-fsck must have the same or more +memory as the Pilosa nodes in the cluster, as it will +"pretend" to be each Pilosa node in turn. However, as each +node's backup is closed before the next node's backup is +opened, we do not require substantially more memory than a single +production node. Short Blake3 cryptographic checksums are +computed for each Roaring fragment and each Key translation +database. These are held in memory (and printed to the log) +for comparing nodes. This comparison forms the heart of +the consistency checks, and is the basis for any subsequent +repair. + +We recommend capturing both stdout and stderr to a log. +Use `&> log` or `2>&1 > log` at the end of the +pilosa-fsck invocation to save a log of the run to disk. + +In a typical cluster, the Replication factor R may be less +than the number of nodes N in the cluster. For example, while +N may be 4, the R may be only 3. In this example, within +each replicated shard, one node will be the primary for +that shard, two nodes will be non-primary replicas, and one +node will be a non-replica. Note that the designation +of primary changes for different Roaring shards within an index, +even on a single node. + +The essence of the the -fix repair operation that pilosa-fsck +can do is this: it will copy from the primary to the +the non-primary replicas. Further, it will remove data from +any non-replica node if it was mistakenly present. + +The pilosa-fsck output log will contain +a sequence of command line 'cp' and 'rm' commands. +These commands are merely a record (with +accompanying justifcation in the comment following the +command) of what actions would be performed to repair +the Roaring file data. + +Only with -fix will the repair actions actually happen +during the pilosa-fsck run. + + +Details: running pilosa-fsck +---------------------------- + +Errors in invocation are reported on stderr and the program will exit with a non-zero +error code if invocation errors are present. A non-zero error code +is returned if a repair is needed and -fix was not given. + +A -fix run will return a zero error code to the shell if the fix was +successfully made; or if no fix was required. + +The log of the run is printed to stdout. + +The -h flag to pilosa-fsck prints a summary of its operation +and a guide to laying out the backup directories. + +The help is reproduced below. + +~~~ +$ pilosa-fsck version: Molecula Pilosa v2.2.1-43-g9dacbccf (Oct 5 2020 1:28PM, 9dacbccf) + +Use: pilosa-fsck -replicas R {-fix} {-q} /backup/1/.pilosa /backup/2/.pilosa ... /backup/N/.pilosa + + -fix + (warning: alters the backed-up node images on disk) copy primary data to replicas to create a consistent cluster. + + -replicas R + (required) R is a positive integer, giving the replicaN or replicator factor for the cluster. This is + the number of replicas maintained in the cluster. Must be the same as the + [cluster] 'replicas = R' entry shared across all the pilosa.conf files on each node. + + -q + be very quiet during analysis and repair + + +Welcome to pilosa-fsck. This is a scan and repair +tool that is modeled after the classic unix file +system utility fsck. + +WARNING: DO NOT RUN ON A LIVE SYSTEM. + +The most important point to remember is that analysis +and repair must be done *offline*. + +Just as fsck must be run on an unmounted disk, +pilosa-fsck must be run on a backup. It must +not be run on the directories where a live Pilosa system +is serving queries. Instead, take a backup first. +A backup is a set of N Pilosa data directories that have been +copied from your live system. They must all +be visible and mounted on one filesystem together. + +pilosa-fsck can be run in scan-mode (without -fix), +or in repair-mode with -fix. The console output +supplies a log documenting the analysis +and showing what data changes would have been made. + +REQUIRED COMMAND LINE ARGUMENTS + +The paths to all the top-level Pilosa +data directories in a cluster must be given on the command +line. The -replicas R flag is also always required. It +must be correct for your cluser. Here R is the same as +the [cluster] stanza "replicas = R" line from your +pilosa.conf. + +Example: + +Suppose you are ready to run pilosa-fsck: +you have taken a backup of your four node Pilosa +cluster and stored it all on one filesystem with +all nodes visible and uncompressed. This +is a pre-requisite to running pilosa-fsck. +Let's suppose we have replication R = 3 set. +In this example, have stored our backed-up directories in + +/backup/molecula + +and the four node backups are in +subdirectories node1/ node2/ node3/ node4/ under this: + +/backup/molecula/node1/ +/backup/molecula/node1/.pilosa/.id +/backup/molecula/node1/.pilosa/.topology +/backup/molecula/node1/.pilosa/myindex + +/backup/molecula/node2/ +/backup/molecula/node2/.pilosa/.id +/backup/molecula/node2/.pilosa/.topology +/backup/molecula/node2/.pilosa/myindex + +/backup/molecula/node3/ +/backup/molecula/node3/.pilosa/.id +/backup/molecula/node3/.pilosa/.topology +/backup/molecula/node3/.pilosa/myindex + +/backup/molecula/node4/ +/backup/molecula/node4/.pilosa/.id +/backup/molecula/node4/.pilosa/.topology +/backup/molecula/node4/.pilosa/myindex + +NOTE: your .pilosa directories need not be named .pilosa. They can +be something else, such as when the -d flag to pilosa server was used. +The .id file, the .topology file, and the index directories must be +found directly underneath. + +Then a typical invocation to scan a cluster backup for issues: + +$ cd /backup/molecula/ +$ pilosa-fsck -replicas 3 node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log + +A typical invocation to repair the replication in the same backup: + +$ pilosa-fsck -replicas 3 -fix node1/.pilosa node2/.pilosa node3/.pilosa node4/.pilosa &> log + +In both cases, the .id and .topology files must +be present in the backups. + +Without -fix, no modifications will be made to the backups. Only +by running with -fix will repairs be made. The user can safely +always run with -fix to repair only if needed. + +A zero error code will be returned to the shell if no repairs were needed. + +A zero error code will be also be returned to the shell if +repairs were needed and they were accomplished under -fix. + +A non-zero error code indicates that repairs were needed but +were not made. + +~~~ diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz b/cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz new file mode 100644 index 000000000..28b08adbd Binary files /dev/null and b/cmd/pilosa-fsck/release-pilosa-fsck/backups.tar.gz differ diff --git a/cmd/pilosa-fsck/release-pilosa-fsck/example.sh b/cmd/pilosa-fsck/release-pilosa-fsck/example.sh new file mode 100755 index 000000000..79fd4cf11 --- /dev/null +++ b/cmd/pilosa-fsck/release-pilosa-fsck/example.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set +x +export PATH=.:${PATH} + +# unpack the sample Molecula Pilosa cluster. +tar xf backups.tar.gz + + +# check if repair is needed. +pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa + + +# yes, so do the repairs. This can be done first (only) as well. +# +pilosa-fsck -fix -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa + + +# check again if you like +# +pilosa-fsck -replicas 3 backups/node0/pilosa backups/node1/pilosa backups/node2/pilosa backups/node3/pilosa diff --git a/cmd/pilosa-fsck/vprint.go b/cmd/pilosa-fsck/vprint.go new file mode 100644 index 000000000..83b1681f7 --- /dev/null +++ b/cmd/pilosa-fsck/vprint.go @@ -0,0 +1,177 @@ +// home: https://github.com/glycerine/vprint +// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. +// License: MIT +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package main + +import ( + "fmt" + "io" + "os" + "path" + "runtime" + "runtime/debug" + "sync" + "time" +) + +const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" +const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" + +// for tons of debug output +var VerboseVerbose bool = false + +// convience functions for . import +var pp = PP +var vv = VV + +var panicOn = PanicOn + +func init() { + // keeper linter happy + _ = pp + _ = vv +} + +func PanicOn(err error) { + if err != nil { + panic(err) + } +} + +func PP(format string, a ...interface{}) { + if VerboseVerbose { + TSPrintf(format, a...) + } +} + +func VV(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +func AlwaysPrintf(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +var tsPrintfMut sync.Mutex + +// time-stamped printf +func TSPrintf(format string, a ...interface{}) { + tsPrintfMut.Lock() + Printf("# %s %s ", FileLine(3), ts()) + Printf(format+"\n", a...) + tsPrintfMut.Unlock() +} + +// get timestamp for logging purposes +func ts() string { + return time.Now().Format(RFC3339UsecTz0) +} + +// so we can multi write easily, use our own printf +var OurStdout io.Writer = os.Stdout + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(OurStdout, format, a...) +} + +func FileLine(depth int) string { + _, fileName, fileLine, ok := runtime.Caller(depth) + var s string + if ok { + s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) + } else { + s = "" + } + return s +} + +func stack() string { + return string(debug.Stack()) +} + +func FileExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return false + } + return true +} + +func DirExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return true + } + return false +} + +func FileSize(name string) int64 { + fi, err := os.Stat(name) + if err != nil { + return 0 + } + return fi.Size() +} + +// Caller returns the name of the calling function. +func Caller(upStack int) string { + // elide ourself and runtime.Callers + target := upStack + 2 + + pc := make([]uintptr, target+2) + n := runtime.Callers(0, pc) + + f := runtime.Frame{Function: "unknown"} + if n > 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} + +// happy linter: +var _ = DirExists +var _ = FileExists +var _ = Caller +var _ = stack +var _ = RFC3339MsecTz0 +var _ = RFC3339UsecTz0 +var _ = AlwaysPrintf +var _ = FileSize diff --git a/cmd/random-query/main.go b/cmd/random-query/main.go new file mode 100644 index 000000000..10fdfa6d5 --- /dev/null +++ b/cmd/random-query/main.go @@ -0,0 +1,377 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "flag" + "fmt" + "math/rand" + nethttp "net/http" + "os" + "strings" + "time" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/http" +) + +// RandomQueryConfig +type RandomQueryConfig struct { + + // user facing flags + HostPort string // -hostport + TreeDepth int // -d + QueryCount int // -n + Verbose bool // -v + + IndexMap map[string]*Features + + API *pilosa.API + Info []*pilosa.IndexInfo + + BitmapFunc []string + + Rnd *rand.Rand +} + +type API interface { + + // InternalClient + Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) + Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) + + // API for contrast; just a little different: + //Schema(ctx context.Context) []*IndexInfo + //Query(ctx context.Context, req *pilosa.QueryRequest) (pilosa.QueryResponse, error) +} + +// have to wrap because the ugly little differences between InternalClient and API +type wrapper struct { + api *pilosa.API +} + +func (w *wrapper) Schema(ctx context.Context) ([]*pilosa.IndexInfo, error) { + return w.api.Schema(ctx), nil +} + +func (w *wrapper) Query(ctx context.Context, index string, queryRequest *pilosa.QueryRequest) (*pilosa.QueryResponse, error) { + r, err := w.api.Query(ctx, queryRequest) + return &r, err +} + +func wrapApiToInternalClient(api *pilosa.API) *wrapper { + return &wrapper{api: api} +} + +// call DefineFlags before myflags.Parse() +func (cfg *RandomQueryConfig) DefineFlags(fs *flag.FlagSet) { + fs.StringVar(&cfg.HostPort, "hostport", "localhost:10101", "host:port of pilosa to run random queries on.") + fs.IntVar(&cfg.TreeDepth, "d", 4, "depth of random queries to generate.") + fs.IntVar(&cfg.QueryCount, "n", 100, "number of random queries to generate. Set to 0 for inifinite queries.") + fs.BoolVar(&cfg.Verbose, "v", false, "show queries as they are generated") +} + +// call c.ValidateConfig() after myflags.Parse() +func (c *RandomQueryConfig) ValidateConfig() error { + if c.TreeDepth < 1 { + return fmt.Errorf("-d depth must be 1 or greater; saw %v", c.TreeDepth) + } + if c.QueryCount < 0 { + return fmt.Errorf("-n count must be 0 or greater; saw %v", c.QueryCount) + } + return nil +} + +var ProgramName = "random-query" + +func main() { + + myflags := flag.NewFlagSet(ProgramName, flag.ExitOnError) + cfg := NewRandomQueryConfig() + cfg.DefineFlags(myflags) + + err := myflags.Parse(os.Args[1:]) + if err != nil { + fmt.Fprintf(os.Stderr, "\n%v\n", err.Error()) + os.Exit(1) + } + err = cfg.ValidateConfig() + if err != nil { + fmt.Fprintf(os.Stderr, "%s error: %s\n", ProgramName, err) + os.Exit(1) + } + + err = cfg.Run() + + if err != nil { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + os.Exit(1) + } +} + +func (cfg *RandomQueryConfig) Run() (err error) { + remoteClient := nethttp.DefaultClient + cli, err := http.NewInternalClient(cfg.HostPort, remoteClient) + if err != nil { + return err + } + ctx := context.Background() + totalQ := 0 + loops := 0 + t0 := time.Now() + +NewSetup: + err = cfg.Setup(cli) + if err != nil { + return err + } + + if len(cfg.IndexMap) == 0 { + return fmt.Errorf("no rows to query") + } + + var indexes []string + for index := range cfg.IndexMap { + indexes = append(indexes, index) + } + + for j := 0; ; j++ { + if cfg.QueryCount > 0 { + if j >= cfg.QueryCount { + break + } + } else { + // else keep doing queries forever... + if loops > 0 && loops%500 == 0 { + // ...but account for any new data arrived by getting + // the schema and rows again every so often. + loops++ + goto NewSetup + } + } + if totalQ%100 == 0 { + dur := time.Since(t0) + if dur > 0 { + qps := 1e9 * float64(totalQ) / float64(dur) + AlwaysPrintf("totalQueries run: %v elapsed: %v qps: %0.02f", totalQ, dur, qps) + } + } + + index := indexes[rand.Intn(len(indexes))] + + pql, err := cfg.GenQuery(index) + panicOn(err) + + if cfg.Verbose { + fmt.Printf("pql = '%v'\n", pql) + } + + // Query node0. + res, err := cli.Query(ctx, index, &pilosa.QueryRequest{Index: index, Query: pql}) + if err != nil { + AlwaysPrintf("QUERY FAILED! queries before this=%v; err = '%v', pql='%v'", loops, err, pql) + return err + } + if cfg.Verbose { + fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) + } + totalQ++ + loops++ + } + + return nil +} + +type Features struct { + Slc []IndexFieldRow +} + +func NewRandomQueryConfig() *RandomQueryConfig { + return &RandomQueryConfig{ + IndexMap: make(map[string]*Features), + } +} + +type IndexFieldRow struct { + Index string + Field string + RowID uint64 + RowKey string + IsRowKey bool +} + +// Run a RandomQuery takes a list of RowIDFeatures and ColumnKeyObjects +// and spits back a PQL query +// +func (cfg *RandomQueryConfig) Setup(api API) (err error) { + + ctx := context.Background() + cfg.Info, err = api.Schema(ctx) + if err != nil { + return err + } + for i, ii := range cfg.Info { + _ = i + for k, fld := range ii.Fields { + _ = k + if fld.Options.Type == "set" { + pql := fmt.Sprintf("Rows(%v)", fld.Name) + + res, err := api.Query(ctx, ii.Name, &pilosa.QueryRequest{Index: ii.Name, Query: pql}) + panicOn(err) + if cfg.Verbose { + fmt.Printf("success on pql = '%v'; res='%v'\n", pql, res.Results[0]) + } + // if the option is set to use RowKeys, then must get the Keys instead of the Rows from the RowIdentifiers. + // e.g. + // success on pql = 'Rows(aba)'; res='&pilosa.RowIdentifiers{Rows:[]uint64(nil), Keys:[]string{"aba1", "aba2"} + // success on pql = 'Rows(f)'; res='pilosa.RowIdentifiers{Rows:[]uint64{0x1}, Keys:[]string(nil), field:"f"}' + + switch x := res.Results[0].(type) { + case *pilosa.RowIdentifiers: + // internalClient gets this + cfg.AddResponse(ii.Name, fld.Name, x) + case pilosa.RowIdentifiers: + // test gets this + cfg.AddResponse(ii.Name, fld.Name, &x) + } + } + } + } + + cfg.BitmapFunc = []string{"Union", "Intersect", "Xor", "Not", "Difference"} + seed := int64(42) + cfg.Rnd = rand.New(rand.NewSource(seed)) + + return nil +} + +func (cfg *RandomQueryConfig) AddResponse(index, field string, x *pilosa.RowIdentifiers) { + for _, rowID := range x.Rows { + cfg.AddFeature(index, field, rowID, "", false) + } + for _, rowKey := range x.Keys { + cfg.AddFeature(index, field, 0, rowKey, true) + } +} + +func (cfg *RandomQueryConfig) GenQuery(index string) (pql string, err error) { + + tree := cfg.GenTree(index, cfg.TreeDepth) + + if cfg.Verbose { + fmt.Printf("%v\n", tree.StringIndent(0)) + } + pql = tree.ToPQL() + + // avoid using too much bandwidth, just count the final bitmap. + pql = fmt.Sprintf("Count(%v)", pql) + return +} + +type Tree struct { + Chd []*Tree + + S string +} + +func (tr *Tree) StringIndent(ind int) (s string) { + spc := strings.Repeat(" ", ind) + spc1 := strings.Repeat(" ", ind+1) + var chds []string + leaf := true + if len(tr.Chd) == 0 { + // leaf + } else { + leaf = false + for _, chd := range tr.Chd { + chds = append(chds, chd.StringIndent(ind+1)) + } + } + if leaf { + s += fmt.Sprintf("%v %v\n", spc1, tr.S) + } else { + for i, c := range chds { + if i == 0 { + s += fmt.Sprintf("%v %v\n%v", spc, tr.S, c) + } else { + s += fmt.Sprintf("%v", c) + } + } + } + return +} + +func (cfg *RandomQueryConfig) GenTree(index string, depth int) (tr *Tree) { + if depth == 0 { + slc := cfg.IndexMap[index].Slc + //vv("depth is 0, slc = '%#v'", slc) + r := cfg.Rnd.Intn(len(slc)) + fea := slc[r] + if fea.IsRowKey { + return &Tree{S: fmt.Sprintf("Row(%v='%v')", fea.Field, fea.RowKey)} + } + return &Tree{S: fmt.Sprintf("Row(%v=%v)", fea.Field, fea.RowID)} + } + + r := cfg.Rnd.Intn(len(cfg.BitmapFunc)) + f := cfg.BitmapFunc[r] + tr = &Tree{S: f} + numChild := 2 + switch f { + case "Union", "Intersect", "Xor": + numChild = cfg.Rnd.Intn(8) + 2 + case "Not": + numChild = 1 + case "Difference": + numChild = 2 + } + for i := 0; i < numChild; i++ { + tr.Chd = append(tr.Chd, cfg.GenTree(index, depth-1)) + } + return +} + +func (tr *Tree) ToPQL() (s string) { + + if len(tr.Chd) == 0 { + // leaf + return tr.S + } + + var chds []string + for _, c := range tr.Chd { + chds = append(chds, c.ToPQL()) + } + all := strings.Join(chds, ", ") + return fmt.Sprintf("%v(%v)", tr.S, all) +} + +func (cfg *RandomQueryConfig) AddFeature(index, field string, rowID uint64, rowKey string, isRowKey bool) { + + f, ok := cfg.IndexMap[index] + if !ok { + f = &Features{} + cfg.IndexMap[index] = f + } + f.Slc = append(f.Slc, IndexFieldRow{ + Index: index, + Field: field, + RowID: rowID, + RowKey: rowKey, + IsRowKey: isRowKey, + }) +} diff --git a/cmd/random-query/main_test.go b/cmd/random-query/main_test.go new file mode 100644 index 000000000..646846899 --- /dev/null +++ b/cmd/random-query/main_test.go @@ -0,0 +1,165 @@ +// Copyright 2020 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "math/rand" + "strconv" + "testing" + + "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/boltdb" + "github.com/pilosa/pilosa/v2/http" + "github.com/pilosa/pilosa/v2/server" + "github.com/pilosa/pilosa/v2/test" +) + +func Test_RandomQuery(t *testing.T) { + + cfg := NewRandomQueryConfig() + + nNodes := 1 + nReplicas := 1 + + name := t.Name() + var nodeid []string + for i := 0; i < nNodes; i++ { + // work around a bug in the test.MustRunCluster that corrupts + // the .topology file if we only join name with one "_" underscore. + nodeid = append(nodeid, name+"__"+strconv.Itoa(i)) + } + + c := test.MustRunCluster(t, nNodes, + []server.CommandOption{ + server.OptCommandServerOptions( + pilosa.OptServerNodeID(nodeid[0]), + pilosa.OptServerOpenTranslateStore(boltdb.OpenTranslateStore), + pilosa.OptServerOpenTranslateReader(http.GetOpenTranslateReaderFunc(nil)), + pilosa.OptServerReplicaN(nReplicas), + )}, + ) + defer c.Close() + + var nodes []*test.Command + var dirs []string + for i := 0; i < nNodes; i++ { + nd := c.GetNode(i) + nodes = append(nodes, nd) + dirs = append(dirs, nd.Server.Holder().Path()) + } + _ = dirs + + ctx := context.Background() + + indexes := []string{"rick", "morty"} + fieldName := []string{"f", "flying_car"} + idx := make([]*pilosa.Index, len(indexes)) + field := make([]*pilosa.Field, len(indexes)) + + var err error + + for i := range indexes { + + idx[i], err = nodes[0].API.CreateIndex(ctx, indexes[i], pilosa.IndexOptions{Keys: true, TrackExistence: true}) + if err != nil { + t.Fatalf("creating index: %v", err) + } + if idx[i].CreatedAt() == 0 { + t.Fatal("index createdAt is empty") + } + + field[i], err = nodes[0].API.CreateField(ctx, indexes[i], fieldName[i], pilosa.OptFieldTypeSet(pilosa.DefaultCacheType, 100)) + if err != nil { + t.Fatalf("creating field: %v", err) + } + if field[i].CreatedAt() == 0 { + t.Fatal("field createdAt is empty") + } + } + + timestamp := int64(0) + + for i := range indexes { + + // Generate some keyed records. + rowIDs := []uint64{} + timestamps := []int64{} + N := 10 + for j := 1; j <= N; j++ { + rowIDs = append(rowIDs, uint64(j)) + timestamps = append(timestamps, timestamp) + } + + var colKeys []string + switch i { + case 0: + // Keys are sharded so ordering is not guaranteed. + colKeys = []string{"col10", "col8", "col9", "col6", "col7", "col4", "col5", "col2", "col3", "col1"} + colKeys = colKeys[:N] + case 1: + colKeys = []string{"col11", "col12"} + N = len(colKeys) + rowIDs = rowIDs[:N] + timestamps = timestamps[:N] + } + + // Import data with keys to the coordinator (node0) and verify that it gets + // translated and forwarded to the owner of shard 0 (node1; because of offsetModHasher) + req := &pilosa.ImportRequest{ + Index: indexes[i], + IndexCreatedAt: idx[i].CreatedAt(), + Field: fieldName[i], + FieldCreatedAt: field[i].CreatedAt(), + + // even though this says Shard: 0, that won't matter. The column keys + // get hashed and that decides the actual shard. + Shard: 0, + RowIDs: rowIDs, + ColumnKeys: colKeys, + Timestamps: timestamps, + } + //vv("rowIDs = '%#v'", rowIDs) + //vv("colKeys = '%#v'", colKeys) + + qcx := nodes[0].API.Txf().NewQcx() + + if err := nodes[0].API.Import(ctx, qcx, req); err != nil { + t.Fatal(err) + } + panicOn(qcx.Finish()) + //qcx.Reset() + } + // end of setup. + + panicOn(cfg.Setup(wrapApiToInternalClient(nodes[0].API))) + + for j := 0; j < 4; j++ { + index := indexes[rand.Intn(len(indexes))] + + pql, err := cfg.GenQuery(index) + panicOn(err) + + //vv("pql = '%v'", pql) + + // Query node0. + res, err := nodes[0].API.Query(ctx, &pilosa.QueryRequest{Index: index, Query: pql}) + if err != nil { + t.Fatal(err) + } + _ = res + //vv("success on pql = '%v'; res='%v'", pql, res.Results[0]) + } +} diff --git a/cmd/random-query/vprint.go b/cmd/random-query/vprint.go new file mode 100644 index 000000000..83b1681f7 --- /dev/null +++ b/cmd/random-query/vprint.go @@ -0,0 +1,177 @@ +// home: https://github.com/glycerine/vprint +// Copyright 2019 Jason E. Aten, Ph.D. All rights reserved. +// License: MIT +// +// MIT License +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +package main + +import ( + "fmt" + "io" + "os" + "path" + "runtime" + "runtime/debug" + "sync" + "time" +) + +const RFC3339MsecTz0 = "2006-01-02T15:04:05.000Z07:00" +const RFC3339UsecTz0 = "2006-01-02T15:04:05.000000Z07:00" + +// for tons of debug output +var VerboseVerbose bool = false + +// convience functions for . import +var pp = PP +var vv = VV + +var panicOn = PanicOn + +func init() { + // keeper linter happy + _ = pp + _ = vv +} + +func PanicOn(err error) { + if err != nil { + panic(err) + } +} + +func PP(format string, a ...interface{}) { + if VerboseVerbose { + TSPrintf(format, a...) + } +} + +func VV(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +func AlwaysPrintf(format string, a ...interface{}) { + TSPrintf(format, a...) +} + +var tsPrintfMut sync.Mutex + +// time-stamped printf +func TSPrintf(format string, a ...interface{}) { + tsPrintfMut.Lock() + Printf("# %s %s ", FileLine(3), ts()) + Printf(format+"\n", a...) + tsPrintfMut.Unlock() +} + +// get timestamp for logging purposes +func ts() string { + return time.Now().Format(RFC3339UsecTz0) +} + +// so we can multi write easily, use our own printf +var OurStdout io.Writer = os.Stdout + +// Printf formats according to a format specifier and writes to standard output. +// It returns the number of bytes written and any write error encountered. +func Printf(format string, a ...interface{}) (n int, err error) { + return fmt.Fprintf(OurStdout, format, a...) +} + +func FileLine(depth int) string { + _, fileName, fileLine, ok := runtime.Caller(depth) + var s string + if ok { + s = fmt.Sprintf("%s:%d", path.Base(fileName), fileLine) + } else { + s = "" + } + return s +} + +func stack() string { + return string(debug.Stack()) +} + +func FileExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return false + } + return true +} + +func DirExists(name string) bool { + fi, err := os.Stat(name) + if err != nil { + return false + } + if fi.IsDir() { + return true + } + return false +} + +func FileSize(name string) int64 { + fi, err := os.Stat(name) + if err != nil { + return 0 + } + return fi.Size() +} + +// Caller returns the name of the calling function. +func Caller(upStack int) string { + // elide ourself and runtime.Callers + target := upStack + 2 + + pc := make([]uintptr, target+2) + n := runtime.Callers(0, pc) + + f := runtime.Frame{Function: "unknown"} + if n > 0 { + frames := runtime.CallersFrames(pc[:n]) + for i := 0; i <= target; i++ { + contender, more := frames.Next() + if i == target { + f = contender + } + if !more { + break + } + } + } + return f.Function +} + +// happy linter: +var _ = DirExists +var _ = FileExists +var _ = Caller +var _ = stack +var _ = RFC3339MsecTz0 +var _ = RFC3339UsecTz0 +var _ = AlwaysPrintf +var _ = FileSize diff --git a/ctl/server.go b/ctl/server.go index 968b2bd6d..221f3918f 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -47,6 +47,7 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVarP(&srv.Config.Cluster.ReplicaN, "cluster.replicas", "", 1, "Number of hosts each piece of data should be stored on.") flags.StringSliceVarP(&srv.Config.Cluster.Hosts, "cluster.hosts", "", []string{}, "Comma separated list of hosts in cluster. Only used for testing.") flags.DurationVarP((*time.Duration)(&srv.Config.Cluster.LongQueryTime), "cluster.long-query-time", "", time.Minute, "Duration that will trigger log and stat messages for slow queries.") + flags.StringVar(&srv.Config.Cluster.Name, "cluster.name", srv.Config.Cluster.Name, "Human-readable name for the cluster.") // Translation flags.StringVarP(&srv.Config.Translation.PrimaryURL, "translation.primary-url", "", srv.Config.Translation.PrimaryURL, "DEPRECATED: URL for primary translation node for replication.") diff --git a/dbshard.go b/dbshard.go index c2e0b3dbe..27bc4c6f9 100644 --- a/dbshard.go +++ b/dbshard.go @@ -320,6 +320,9 @@ func (per *DBPerShard) DeleteIndex(index string) (err error) { } } } + // allow the index to be created again anew. + delete(per.dbh.Index, index) + return } @@ -436,7 +439,7 @@ func (per *DBPerShard) GetDBShard(index string, shard uint64, idx *Index) (dbs * if len(per.types) == 1 && per.types[0] == roaringTxn { // roaring txn are nil/fake anyway. Don't freak out. } else { - panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'", dbs)) + panic(fmt.Sprintf("cannot retain closed dbs across holder ReOpen dbs='%p'; per.types[0]='%v'; len(per.types)=%v", dbs, per.types[0], len(per.types))) } } if !ok { diff --git a/executor.go b/executor.go index 774f137fe..89376914f 100644 --- a/executor.go +++ b/executor.go @@ -1074,14 +1074,16 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinct") defer span.Finish() - field := c.Args["field"] - if field == "" { - return SignedRow{}, fmt.Errorf("plugin operation %s(): field required", c.Name) + field, hasField, err := c.StringArg("field") + if err != nil { + return SignedRow{}, errors.Wrap(err, "loading field option in Distinct query") + } else if !hasField { + return SignedRow{}, fmt.Errorf("missing field option in Distinct query") } // Execute calls in bulk on each remote node and merge. mapFn := func(ctx context.Context, shard uint64) (_ interface{}, err error) { - return e.executeDistinctShard(ctx, qcx, index, c, shard) + return e.executeDistinctShard(ctx, qcx, index, field, c, shard) } // Merge returned results at coordinating node. @@ -1098,7 +1100,7 @@ func (e *executor) executeDistinct(ctx context.Context, qcx *Qcx, index string, return SignedRow{}, err } other, _ := result.(SignedRow) - other.field = field.(string) + other.field = field return other, nil } @@ -1385,11 +1387,15 @@ func (e *executor) executeBitmapCallShard(ctx context.Context, qcx *Qcx, index s // executeDistinctShard executes a Distinct call on a single shard, yielding // a SignedRow of the values found. -func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, c *pql.Call, shard uint64) (result SignedRow, err error) { +func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index string, fieldName string, c *pql.Call, shard uint64) (result SignedRow, err error) { span, ctx := tracing.StartSpanFromContext(ctx, "Executor.executeDistinctShard") defer span.Finish() idx := e.Holder.Index(index) + field := e.Holder.Field(index, fieldName) + if field == nil { + return SignedRow{}, ErrFieldNotFound + } var filter *Row var filterBitmap *roaring.Bitmap @@ -1406,13 +1412,6 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str } } - fieldName, _ := c.Args["field"].(string) - - field := e.Holder.Field(index, fieldName) - if field == nil { - return result, nil - } - bsig := field.bsiGroup(fieldName) if bsig == nil { return result, nil @@ -1425,7 +1424,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str tx, finisher := qcx.GetTx(Txo{Write: !writable, Index: idx, Shard: shard}) defer finisher(&err) - existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*0, ShardWidth*1) + existsBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*0, ShardWidth*1) if err != nil { return result, err } @@ -1436,7 +1435,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str return result, nil } - signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*1, ShardWidth*2) + signBitmap, err := tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*1, ShardWidth*2) if err != nil { return result, nil } @@ -1444,7 +1443,7 @@ func (e *executor) executeDistinctShard(ctx context.Context, qcx *Qcx, index str dataBitmaps := make([]*roaring.Bitmap, depth) for i := uint64(0); i < depth; i++ { - dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, 0, ShardWidth*(i+2), ShardWidth*(i+3)) + dataBitmaps[i], err = tx.OffsetRange(index, fieldName, view, shard, ShardWidth*shard, ShardWidth*(i+2), ShardWidth*(i+3)) if err != nil { return result, err } @@ -2593,10 +2592,8 @@ func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string, // in order to represent `Rows` for the field. var views = []string{viewStandard} - // Handle `int` and `time` fields. switch f.Type() { - case FieldTypeInt: - return nil, errors.New("int fields not supported by Rows() query") + case FieldTypeSet, FieldTypeMutex: case FieldTypeTime: var err error @@ -2658,6 +2655,8 @@ func (e *executor) executeRowsShard(ctx context.Context, qcx *Qcx, index string, // Determine the views based on the specified time range. views = viewsByTimeRange(viewStandard, fromTime, toTime, q) } + default: + return nil, errors.Errorf("%s fields not supported by Rows() query", f.Type()) } start := uint64(0) diff --git a/executor_test.go b/executor_test.go index a34041844..e379daf7d 100644 --- a/executor_test.go +++ b/executor_test.go @@ -4851,6 +4851,8 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { defer c.Close() c.CreateField(t, "i", pilosa.IndexOptions{}, "general") c.CreateField(t, "i", pilosa.IndexOptions{}, "integer", pilosa.OptFieldTypeInt(-1000, 1000)) + c.CreateField(t, "i", pilosa.IndexOptions{}, "decimal", pilosa.OptFieldTypeDecimal(2)) + c.CreateField(t, "i", pilosa.IndexOptions{}, "bool", pilosa.OptFieldTypeBool()) tests := []struct { query string @@ -4884,6 +4886,18 @@ func TestExecutor_Execute_Query_Error(t *testing.T) { query: "GroupBy(Rows(integer), prev=-1)", error: "unknown arg 'prev'", }, + { + query: "Rows(integer)", + error: "int fields not supported by Rows() query", + }, + { + query: "Rows(decimal)", + error: "decimal fields not supported by Rows() query", + }, + { + query: "Rows(bool)", + error: "bool fields not supported by Rows() query", + }, } for i, test := range tests { @@ -5293,7 +5307,7 @@ func TestExecutor_ForeignIndex(t *testing.T) { ) // Populate parent data. - c.Query(t, "parent", ` + c.Query(t, "parent", fmt.Sprintf(` Set("one", general=1) Set("two", general=1) Set("three", general=1) @@ -5302,25 +5316,25 @@ func TestExecutor_ForeignIndex(t *testing.T) { Set("twenty-two", general=2) Set("twenty-three", general=2) - Set("one", general=3) - Set("twenty-one", general=3) - `) + Set("one", general=%d) + Set("twenty-one", general=%d) + `, ShardWidth, ShardWidth)) // Populate child data. - c.Query(t, "child", ` + c.Query(t, "child", fmt.Sprintf(` Set(1, parent_id="one") Set(2, parent_id="two") - Set(3, parent_id="one") + Set(%d, parent_id="one") Set(4, parent_id="twenty-one") - `) + `, ShardWidth)) // Populate color data. - c.Query(t, "child", ` + c.Query(t, "child", fmt.Sprintf(` Set(1, color="red") Set(2, color="blue") - Set(3, color="blue") + Set(%d, color="blue") Set(4, color="red") - `) + `, ShardWidth)) distinct := c.Query(t, "child", `Distinct(index="child", field="parent_id")`).Results[0].(pilosa.SignedRow) if !sameStringSlice(distinct.Pos.Keys, []string{"one", "two", "twenty-one"}) { @@ -5328,7 +5342,7 @@ func TestExecutor_ForeignIndex(t *testing.T) { } eq := c.Query(t, "child", `Row(parent_id=="one")`).Results[0].(*pilosa.Row) - if !reflect.DeepEqual(eq.Columns(), []uint64{1, 3}) { + if !reflect.DeepEqual(eq.Columns(), []uint64{1, ShardWidth}) { t.Fatalf("unexpected columns: %v", eq.Columns()) } @@ -5337,7 +5351,7 @@ func TestExecutor_ForeignIndex(t *testing.T) { t.Fatalf("unexpected columns: %v", neq.Columns()) } - join := c.Query(t, "parent", `Intersect(Row(general=3), Distinct(Row(color="blue"), index="child", field="parent_id"))`).Results[0].(*pilosa.Row) + join := c.Query(t, "parent", fmt.Sprintf(`Intersect(Row(general=%d), Distinct(Row(color="blue"), index="child", field="parent_id"))`, ShardWidth)).Results[0].(*pilosa.Row) if !reflect.DeepEqual(join.Keys, []string{"one"}) { t.Fatalf("unexpected keys: %v", join.Keys) } diff --git a/fragment.go b/fragment.go index 122688ec5..33d0f3bfd 100644 --- a/fragment.go +++ b/fragment.go @@ -798,26 +798,28 @@ func (f *fragment) unprotectedSetRow(tx Tx, row *Row, rowID uint64) (changed boo // From the given row, get the rowSegment for this shard. seg := row.segment(f.shard) - if seg == nil { - return changed, nil - } - - // Put each container from rowSegment to fragment storage. - citer, _ := seg.data.Containers.Iterator(f.shard << shardVsContainerExponent) - for citer.Next() { - k, c := citer.Value() - if err := tx.PutContainer(f.index, f.field, f.view, f.shard, headContainerKey+(k%(1< b.PartitionID { return false } - return a.Field < b.Field + if a.Field < b.Field { + return true + } + if a.Field > b.Field { + return false + } + return a.NodeID < b.NodeID }) } // sums is only guaranteed to be sorted by (index, PartitionID, field) iff err returns nil -func (i *Index) ComputeTranslatorSummary(verbose bool) (ats *AllTranslatorSummary, err error) { - i.mu.RLock() - defer i.mu.RUnlock() +func (idx *Index) ComputeTranslatorSummary(verbose, checkKeys, applyKeyRepairs bool, topo *Topology, nodeID string, parallelReaders int) (ats *AllTranslatorSummary, err error) { + idx.mu.RLock() + defer idx.mu.RUnlock() ats = &AllTranslatorSummary{} + var atsMu sync.Mutex - fmt.Printf("\nindex: %v\n=================\n", i.name) - for _, fld := range i.fields { - sum, err := fld.translateStore.ComputeTranslatorSummary() - if err != nil { - return ats, err - } - sum.Field = fld.name - sum.Index = i.Name() - sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, i.Name()))) - - if verbose { - fmt.Printf("row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name) - } - ats.Sums = append(ats.Sums, sum) + if verbose { + fmt.Printf("\n# index: %v\n# =================\n", idx.name) } - fmt.Printf("====================\n") + jobQ := make(chan func() error, 10000) + var errmu sync.Mutex - for partitionID, store := range i.translateStores { - sum, err := store.ComputeTranslatorSummary() - if err != nil { - return ats, err - } - if sum == nil { - // probably one of the Noop stores - continue - } - sum.PartitionID = partitionID - sum.Index = i.Name() - - sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, i.Name()))) - if verbose { - fmt.Printf("col blake3-%v keyN: %10v idN: %10v paritionID: %03v \n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID) - } - ats.Sums = append(ats.Sums, sum) + if parallelReaders < 1 { + // turn it up to 11 + parallelReaders = 10000 } + + halters := make([]*idem.Halter, parallelReaders) + for j := 0; j < parallelReaders; j++ { + h := idem.NewHalter() + halters[j] = h + } + for _, h := range halters { + go func(h *idem.Halter) { + defer h.MarkDone() + for { + select { + case <-h.ReqStop.Chan: + return + case f, ok := <-jobQ: + if !ok || f == nil { + // channel closed, finish up + return + } + + err1 := f() + if err1 != nil { + errmu.Lock() + if err == nil { + err = err1 + } + errmu.Unlock() + // an error occurred, tell everyone to stop + for _, h2 := range halters { + h2.RequestStop() + } + return + } + } + } + }(h) + } + +floop: + for _, fld := range idx.fields { + fld := fld + + fun := func() error { + //vv("ComputeTranslatorSummary() on fld '%v'", fld.name) + sum, err := fld.translateStore.ComputeTranslatorSummaryRows() + if err != nil { + return err + } + sum.Field = fld.name + sum.Index = idx.Name() + sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, fld.name, idx.Name()))) + sum.IsColKey = false + if verbose { + fmt.Printf("# row blake3-%v keyN: %5v idN: %5v field: '%v'\n", sum.Checksum, sum.KeyCount, sum.IDCount, fld.name) + } + atsMu.Lock() + ats.Sums = append(ats.Sums, sum) + atsMu.Unlock() + return nil + } + + select { + case <-halters[0].ReqStop.Chan: + break floop + case jobQ <- fun: + } + } // end floop + + if verbose { + fmt.Printf("# ====================\n") + } + +tloop: + for partitionID, store := range idx.translateStores { + partitionID := partitionID + store := store + + fun2 := func() error { + //vv("ComputeTranslatorSummary() running on store.Path = '%v'", store.GetStorePath()) + if checkKeys { + prim := topo.PrimaryNodeIndex(partitionID) + primID := topo.nodeIDs[prim] + + // note: we fix irrespective of nodeID == primID now, so that we + // get a fine grain report of what maps were off. + + if verbose { + // This is pilosa-fsck output, not regular log. + fmt.Printf("# doing analysis of keys on nodeID '%v', and primID '%v'\n", nodeID, primID) + } + changed, err := store.RepairKeys(topo, verbose, applyKeyRepairs) + if err != nil { + return errors.Wrap(err, "ComputeTranslatorSummary() call to store.Repair()") + } + if changed { + atsMu.Lock() + ats.RepairNeeded = true + atsMu.Unlock() + } + } + + // key repair has to be above, because we compute the checksum below. + + sum, err := store.ComputeTranslatorSummaryCols(partitionID, topo) + if err != nil { + return err + } + if sum == nil { + // probably one of the Noop stores from the tests. + return nil + } + sum.IsColKey = true + sum.PartitionID = partitionID + sum.Index = idx.Name() + sum.StorePath = store.GetStorePath() + sum.NodeID = nodeID + sum.IsPrimary = topo.IsPrimary(nodeID, partitionID) + + replicas := topo.GetNonPrimaryReplicas(partitionID) + for _, replica := range replicas { + if nodeID == replica { + sum.IsReplica = true + break + } + } + + sum.Checksum = hash.Blake3sum16([]byte(fmt.Sprintf("%v/%v/%v", sum.Checksum, partitionID, idx.Name()))) + if verbose { + // This is not regular index logging. This is output of the pilosa-fsck tool. + // So it must be printing straight to stdout. + fmt.Printf("# col blake3-%v keyN: %10v idN: %10v paritionID: %03v primary: %03v\n", sum.Checksum, sum.KeyCount, sum.IDCount, partitionID, sum.PrimaryNodeIndex) + } + atsMu.Lock() + ats.Sums = append(ats.Sums, sum) + atsMu.Unlock() + + return nil + } + select { + case <-halters[0].ReqStop.Chan: + break tloop + case jobQ <- fun2: + } + } // end tloop + + close(jobQ) // tell the workers no more jobs. + + // wait for everyone to finish + for _, h := range halters { + <-h.Done.Chan + } + return ats, err +} + +// returned by WriteFragmentChecksums +type IndexFragmentSummary struct { + Dir string + NodeID string + Index string + IndexPath string + Frg []*FragSum + + RelPath2fsum map[string]*FragSum +} + +func (ifs *IndexFragmentSummary) String() (s string) { + s = fmt.Sprintf(`&pilosa.IndexFragmentSummary{ + Dir: '%v' + NodeID: '%v' + Index: '%v' + IndexPath: '%v' +`, ifs.Dir, ifs.NodeID, ifs.Index, ifs.IndexPath) + for _, frg := range ifs.Frg { + s += frg.String() + "\n" + } + s += "}\n" return } -func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool) { +// used in IndexFragmentSummary +type FragSum struct { + AbsPath string + RelPath string + + // critically, NodeID is how pilosa-fsck figures out if this + // fragment should be deleted if it is on a node it should not be. + NodeID string + + Index string + Field string + View string + Shard uint64 + Hotbits int + Checksum string + Primary int + + ScanDone bool // pilosa-fsck will set this once done to avoid repairing multiple times. +} + +func (fsum *FragSum) String() (s string) { + return fmt.Sprintf("%#v", fsum) +} + +// if verbose, then print to w. +func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool, topo *Topology, verbose bool) (sum *IndexFragmentSummary) { + sum = &IndexFragmentSummary{ + Index: idx.name, + IndexPath: idx.path, + RelPath2fsum: make(map[string]*FragSum), + } paths, err := listFilesUnderDir(idx.path, false, "", true) panicOn(err) index := idx.name @@ -822,14 +1025,37 @@ func (idx *Index) WriteFragmentChecksums(w io.Writer, showBits, showOps bool) { continue // ignore .meta paths } abspath := idx.path + sep + relpath + primary := topo.GetPrimaryForShardReplication(index, shard) checksum, hotbits := RoaringFragmentChecksum(abspath, index, field, view, shard) - fmt.Fprintf(w, "frg blake3-%v field: '%v' view: '%v' shard: %3v hotbits: %10v\n", checksum, field, view, shard, hotbits) + if verbose { + fmt.Fprintf(w, "# frg blake3-%v field: '%v' view: '%v' shard: %3v hotbits: %10v primary:%03v\n", checksum, field, view, shard, hotbits, primary) + } + fsum := &FragSum{ + AbsPath: abspath, + RelPath: relpath, + Index: index, + Field: field, + View: view, + Shard: shard, + Hotbits: hotbits, + Checksum: checksum, + Primary: primary, + } + sum.Frg = append(sum.Frg, fsum) + _, already := sum.RelPath2fsum[relpath] + if already { + panic(fmt.Sprintf("relpath '%v' was already present!?!", relpath)) + } + sum.RelPath2fsum[relpath] = fsum n++ } if n == 0 { - fmt.Fprintf(w, "empty index '%v'", idx.path) + if verbose { + fmt.Fprintf(w, "empty index '%v'", idx.path) + } } + return } func (idx *Index) Txf() *TxFactory { diff --git a/license.exceptions b/license.exceptions index ffb08a26e..46230d311 100644 --- a/license.exceptions +++ b/license.exceptions @@ -19,3 +19,7 @@ ./cmd/pilosa-keydump/vprint.go ./cmd/pilosa-keydump/keydump.go ./synthload/vprint.go +./proto/vdsm/vdsm.proto +./proto/vdsm/vdsm.pb.go +./cmd/pilosa-fsck/vprint.go +./cmd/random-query/vprint.go diff --git a/mock/translator.go b/mock/translator.go index e577baf12..6afd0d48e 100644 --- a/mock/translator.go +++ b/mock/translator.go @@ -37,7 +37,10 @@ type TranslateStore struct { EntryReaderFunc func(ctx context.Context, offset uint64) (pilosa.TranslateEntryReader, error) } -func (s *TranslateStore) ComputeTranslatorSummary() (sum *pilosa.TranslatorSummary, err error) { +func (s *TranslateStore) ComputeTranslatorSummaryRows() (sum *pilosa.TranslatorSummary, err error) { + return +} +func (s *TranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *pilosa.Topology) (sum *pilosa.TranslatorSummary, err error) { return } @@ -93,6 +96,14 @@ func (s *TranslateStore) ReadFrom(r io.Reader) (int64, error) { return 0, nil } +func (s *TranslateStore) RepairKeys(topo *pilosa.Topology, verbose, applyKeyRepairs bool) (changed bool, err error) { + return +} + +func (s *TranslateStore) GetStorePath() string { + return "" +} + var _ pilosa.TranslateEntryReader = (*TranslateEntryReader)(nil) type TranslateEntryReader struct { @@ -107,3 +118,10 @@ func (r *TranslateEntryReader) Close() error { func (r *TranslateEntryReader) ReadEntry(entry *pilosa.TranslateEntry) error { return r.ReadEntryFunc(entry) } + +func (s *TranslateStore) KeyWalker(walk func(key string, col uint64)) error { + panic("TODO") +} +func (s *TranslateStore) IDWalker(walk func(key string, col uint64)) error { + panic("TODO") +} diff --git a/proto/interface.go b/proto/interface.go index af2599707..77ba3fd8d 100644 --- a/proto/interface.go +++ b/proto/interface.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package pilosa +package proto import ( "fmt" diff --git a/proto/pilosa.pb.go b/proto/pilosa.pb.go index a48ae519b..973b6e173 100644 --- a/proto/pilosa.pb.go +++ b/proto/pilosa.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // source: pilosa.proto -package pilosa +package proto import ( context "context" @@ -24,349 +24,6 @@ var _ = math.Inf // proto package needs to be updated. const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package -type VDS struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *VDS) Reset() { *m = VDS{} } -func (m *VDS) String() string { return proto.CompactTextString(m) } -func (*VDS) ProtoMessage() {} -func (*VDS) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{0} -} - -func (m *VDS) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_VDS.Unmarshal(m, b) -} -func (m *VDS) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_VDS.Marshal(b, m, deterministic) -} -func (m *VDS) XXX_Merge(src proto.Message) { - xxx_messageInfo_VDS.Merge(m, src) -} -func (m *VDS) XXX_Size() int { - return xxx_messageInfo_VDS.Size(m) -} -func (m *VDS) XXX_DiscardUnknown() { - xxx_messageInfo_VDS.DiscardUnknown(m) -} - -var xxx_messageInfo_VDS proto.InternalMessageInfo - -func (m *VDS) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -type GetVDSsRequest struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetVDSsRequest) Reset() { *m = GetVDSsRequest{} } -func (m *GetVDSsRequest) String() string { return proto.CompactTextString(m) } -func (*GetVDSsRequest) ProtoMessage() {} -func (*GetVDSsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{1} -} - -func (m *GetVDSsRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetVDSsRequest.Unmarshal(m, b) -} -func (m *GetVDSsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetVDSsRequest.Marshal(b, m, deterministic) -} -func (m *GetVDSsRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetVDSsRequest.Merge(m, src) -} -func (m *GetVDSsRequest) XXX_Size() int { - return xxx_messageInfo_GetVDSsRequest.Size(m) -} -func (m *GetVDSsRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetVDSsRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_GetVDSsRequest proto.InternalMessageInfo - -type GetVDSsResponse struct { - Vdss []*VDS `protobuf:"bytes,1,rep,name=vdss,proto3" json:"vdss,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetVDSsResponse) Reset() { *m = GetVDSsResponse{} } -func (m *GetVDSsResponse) String() string { return proto.CompactTextString(m) } -func (*GetVDSsResponse) ProtoMessage() {} -func (*GetVDSsResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{2} -} - -func (m *GetVDSsResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetVDSsResponse.Unmarshal(m, b) -} -func (m *GetVDSsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetVDSsResponse.Marshal(b, m, deterministic) -} -func (m *GetVDSsResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetVDSsResponse.Merge(m, src) -} -func (m *GetVDSsResponse) XXX_Size() int { - return xxx_messageInfo_GetVDSsResponse.Size(m) -} -func (m *GetVDSsResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetVDSsResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_GetVDSsResponse proto.InternalMessageInfo - -func (m *GetVDSsResponse) GetVdss() []*VDS { - if m != nil { - return m.Vdss - } - return nil -} - -type GetVDSRequest struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetVDSRequest) Reset() { *m = GetVDSRequest{} } -func (m *GetVDSRequest) String() string { return proto.CompactTextString(m) } -func (*GetVDSRequest) ProtoMessage() {} -func (*GetVDSRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{3} -} - -func (m *GetVDSRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetVDSRequest.Unmarshal(m, b) -} -func (m *GetVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetVDSRequest.Marshal(b, m, deterministic) -} -func (m *GetVDSRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetVDSRequest.Merge(m, src) -} -func (m *GetVDSRequest) XXX_Size() int { - return xxx_messageInfo_GetVDSRequest.Size(m) -} -func (m *GetVDSRequest) XXX_DiscardUnknown() { - xxx_messageInfo_GetVDSRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_GetVDSRequest proto.InternalMessageInfo - -func (m *GetVDSRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -type GetVDSResponse struct { - Vds *VDS `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *GetVDSResponse) Reset() { *m = GetVDSResponse{} } -func (m *GetVDSResponse) String() string { return proto.CompactTextString(m) } -func (*GetVDSResponse) ProtoMessage() {} -func (*GetVDSResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{4} -} - -func (m *GetVDSResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_GetVDSResponse.Unmarshal(m, b) -} -func (m *GetVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_GetVDSResponse.Marshal(b, m, deterministic) -} -func (m *GetVDSResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetVDSResponse.Merge(m, src) -} -func (m *GetVDSResponse) XXX_Size() int { - return xxx_messageInfo_GetVDSResponse.Size(m) -} -func (m *GetVDSResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetVDSResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_GetVDSResponse proto.InternalMessageInfo - -func (m *GetVDSResponse) GetVds() *VDS { - if m != nil { - return m.Vds - } - return nil -} - -type PostVDSRequest struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - Keys bool `protobuf:"varint,2,opt,name=keys,proto3" json:"keys,omitempty"` - TrackExistence bool `protobuf:"varint,3,opt,name=trackExistence,proto3" json:"trackExistence,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PostVDSRequest) Reset() { *m = PostVDSRequest{} } -func (m *PostVDSRequest) String() string { return proto.CompactTextString(m) } -func (*PostVDSRequest) ProtoMessage() {} -func (*PostVDSRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{5} -} - -func (m *PostVDSRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PostVDSRequest.Unmarshal(m, b) -} -func (m *PostVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PostVDSRequest.Marshal(b, m, deterministic) -} -func (m *PostVDSRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_PostVDSRequest.Merge(m, src) -} -func (m *PostVDSRequest) XXX_Size() int { - return xxx_messageInfo_PostVDSRequest.Size(m) -} -func (m *PostVDSRequest) XXX_DiscardUnknown() { - xxx_messageInfo_PostVDSRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_PostVDSRequest proto.InternalMessageInfo - -func (m *PostVDSRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -func (m *PostVDSRequest) GetKeys() bool { - if m != nil { - return m.Keys - } - return false -} - -func (m *PostVDSRequest) GetTrackExistence() bool { - if m != nil { - return m.TrackExistence - } - return false -} - -type PostVDSResponse struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *PostVDSResponse) Reset() { *m = PostVDSResponse{} } -func (m *PostVDSResponse) String() string { return proto.CompactTextString(m) } -func (*PostVDSResponse) ProtoMessage() {} -func (*PostVDSResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{6} -} - -func (m *PostVDSResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_PostVDSResponse.Unmarshal(m, b) -} -func (m *PostVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_PostVDSResponse.Marshal(b, m, deterministic) -} -func (m *PostVDSResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_PostVDSResponse.Merge(m, src) -} -func (m *PostVDSResponse) XXX_Size() int { - return xxx_messageInfo_PostVDSResponse.Size(m) -} -func (m *PostVDSResponse) XXX_DiscardUnknown() { - xxx_messageInfo_PostVDSResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_PostVDSResponse proto.InternalMessageInfo - -type DeleteVDSRequest struct { - Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *DeleteVDSRequest) Reset() { *m = DeleteVDSRequest{} } -func (m *DeleteVDSRequest) String() string { return proto.CompactTextString(m) } -func (*DeleteVDSRequest) ProtoMessage() {} -func (*DeleteVDSRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{7} -} - -func (m *DeleteVDSRequest) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DeleteVDSRequest.Unmarshal(m, b) -} -func (m *DeleteVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DeleteVDSRequest.Marshal(b, m, deterministic) -} -func (m *DeleteVDSRequest) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteVDSRequest.Merge(m, src) -} -func (m *DeleteVDSRequest) XXX_Size() int { - return xxx_messageInfo_DeleteVDSRequest.Size(m) -} -func (m *DeleteVDSRequest) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteVDSRequest.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteVDSRequest proto.InternalMessageInfo - -func (m *DeleteVDSRequest) GetName() string { - if m != nil { - return m.Name - } - return "" -} - -type DeleteVDSResponse struct { - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` -} - -func (m *DeleteVDSResponse) Reset() { *m = DeleteVDSResponse{} } -func (m *DeleteVDSResponse) String() string { return proto.CompactTextString(m) } -func (*DeleteVDSResponse) ProtoMessage() {} -func (*DeleteVDSResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{8} -} - -func (m *DeleteVDSResponse) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_DeleteVDSResponse.Unmarshal(m, b) -} -func (m *DeleteVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_DeleteVDSResponse.Marshal(b, m, deterministic) -} -func (m *DeleteVDSResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_DeleteVDSResponse.Merge(m, src) -} -func (m *DeleteVDSResponse) XXX_Size() int { - return xxx_messageInfo_DeleteVDSResponse.Size(m) -} -func (m *DeleteVDSResponse) XXX_DiscardUnknown() { - xxx_messageInfo_DeleteVDSResponse.DiscardUnknown(m) -} - -var xxx_messageInfo_DeleteVDSResponse proto.InternalMessageInfo - type QueryPQLRequest struct { Index string `protobuf:"bytes,1,opt,name=index,proto3" json:"index,omitempty"` Pql string `protobuf:"bytes,2,opt,name=pql,proto3" json:"pql,omitempty"` @@ -379,7 +36,7 @@ func (m *QueryPQLRequest) Reset() { *m = QueryPQLRequest{} } func (m *QueryPQLRequest) String() string { return proto.CompactTextString(m) } func (*QueryPQLRequest) ProtoMessage() {} func (*QueryPQLRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{9} + return fileDescriptor_ef0691a44d1e275c, []int{0} } func (m *QueryPQLRequest) XXX_Unmarshal(b []byte) error { @@ -425,7 +82,7 @@ func (m *QuerySQLRequest) Reset() { *m = QuerySQLRequest{} } func (m *QuerySQLRequest) String() string { return proto.CompactTextString(m) } func (*QuerySQLRequest) ProtoMessage() {} func (*QuerySQLRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{10} + return fileDescriptor_ef0691a44d1e275c, []int{1} } func (m *QuerySQLRequest) XXX_Unmarshal(b []byte) error { @@ -465,7 +122,7 @@ func (m *StatusError) Reset() { *m = StatusError{} } func (m *StatusError) String() string { return proto.CompactTextString(m) } func (*StatusError) ProtoMessage() {} func (*StatusError) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{11} + return fileDescriptor_ef0691a44d1e275c, []int{2} } func (m *StatusError) XXX_Unmarshal(b []byte) error { @@ -513,7 +170,7 @@ func (m *RowResponse) Reset() { *m = RowResponse{} } func (m *RowResponse) String() string { return proto.CompactTextString(m) } func (*RowResponse) ProtoMessage() {} func (*RowResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{12} + return fileDescriptor_ef0691a44d1e275c, []int{3} } func (m *RowResponse) XXX_Unmarshal(b []byte) error { @@ -566,7 +223,7 @@ func (m *Row) Reset() { *m = Row{} } func (m *Row) String() string { return proto.CompactTextString(m) } func (*Row) ProtoMessage() {} func (*Row) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{13} + return fileDescriptor_ef0691a44d1e275c, []int{4} } func (m *Row) XXX_Unmarshal(b []byte) error { @@ -607,7 +264,7 @@ func (m *TableResponse) Reset() { *m = TableResponse{} } func (m *TableResponse) String() string { return proto.CompactTextString(m) } func (*TableResponse) ProtoMessage() {} func (*TableResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{14} + return fileDescriptor_ef0691a44d1e275c, []int{5} } func (m *TableResponse) XXX_Unmarshal(b []byte) error { @@ -661,7 +318,7 @@ func (m *ColumnInfo) Reset() { *m = ColumnInfo{} } func (m *ColumnInfo) String() string { return proto.CompactTextString(m) } func (*ColumnInfo) ProtoMessage() {} func (*ColumnInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{15} + return fileDescriptor_ef0691a44d1e275c, []int{6} } func (m *ColumnInfo) XXX_Unmarshal(b []byte) error { @@ -717,7 +374,7 @@ func (m *ColumnResponse) Reset() { *m = ColumnResponse{} } func (m *ColumnResponse) String() string { return proto.CompactTextString(m) } func (*ColumnResponse) ProtoMessage() {} func (*ColumnResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{16} + return fileDescriptor_ef0691a44d1e275c, []int{7} } func (m *ColumnResponse) XXX_Unmarshal(b []byte) error { @@ -893,7 +550,7 @@ func (m *Decimal) Reset() { *m = Decimal{} } func (m *Decimal) String() string { return proto.CompactTextString(m) } func (*Decimal) ProtoMessage() {} func (*Decimal) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{17} + return fileDescriptor_ef0691a44d1e275c, []int{8} } func (m *Decimal) XXX_Unmarshal(b []byte) error { @@ -944,7 +601,7 @@ func (m *InspectRequest) Reset() { *m = InspectRequest{} } func (m *InspectRequest) String() string { return proto.CompactTextString(m) } func (*InspectRequest) ProtoMessage() {} func (*InspectRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{18} + return fileDescriptor_ef0691a44d1e275c, []int{9} } func (m *InspectRequest) XXX_Unmarshal(b []byte) error { @@ -1018,7 +675,7 @@ func (m *Uint64Array) Reset() { *m = Uint64Array{} } func (m *Uint64Array) String() string { return proto.CompactTextString(m) } func (*Uint64Array) ProtoMessage() {} func (*Uint64Array) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{19} + return fileDescriptor_ef0691a44d1e275c, []int{10} } func (m *Uint64Array) XXX_Unmarshal(b []byte) error { @@ -1057,7 +714,7 @@ func (m *StringArray) Reset() { *m = StringArray{} } func (m *StringArray) String() string { return proto.CompactTextString(m) } func (*StringArray) ProtoMessage() {} func (*StringArray) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{20} + return fileDescriptor_ef0691a44d1e275c, []int{11} } func (m *StringArray) XXX_Unmarshal(b []byte) error { @@ -1099,7 +756,7 @@ func (m *IdsOrKeys) Reset() { *m = IdsOrKeys{} } func (m *IdsOrKeys) String() string { return proto.CompactTextString(m) } func (*IdsOrKeys) ProtoMessage() {} func (*IdsOrKeys) Descriptor() ([]byte, []int) { - return fileDescriptor_ef0691a44d1e275c, []int{21} + return fileDescriptor_ef0691a44d1e275c, []int{12} } func (m *IdsOrKeys) XXX_Unmarshal(b []byte) error { @@ -1166,15 +823,6 @@ func (*IdsOrKeys) XXX_OneofWrappers() []interface{} { } func init() { - proto.RegisterType((*VDS)(nil), "pilosa.VDS") - proto.RegisterType((*GetVDSsRequest)(nil), "pilosa.GetVDSsRequest") - proto.RegisterType((*GetVDSsResponse)(nil), "pilosa.GetVDSsResponse") - proto.RegisterType((*GetVDSRequest)(nil), "pilosa.GetVDSRequest") - proto.RegisterType((*GetVDSResponse)(nil), "pilosa.GetVDSResponse") - proto.RegisterType((*PostVDSRequest)(nil), "pilosa.PostVDSRequest") - proto.RegisterType((*PostVDSResponse)(nil), "pilosa.PostVDSResponse") - proto.RegisterType((*DeleteVDSRequest)(nil), "pilosa.DeleteVDSRequest") - proto.RegisterType((*DeleteVDSResponse)(nil), "pilosa.DeleteVDSResponse") proto.RegisterType((*QueryPQLRequest)(nil), "pilosa.QueryPQLRequest") proto.RegisterType((*QuerySQLRequest)(nil), "pilosa.QuerySQLRequest") proto.RegisterType((*StatusError)(nil), "pilosa.StatusError") @@ -1193,64 +841,54 @@ func init() { func init() { proto.RegisterFile("pilosa.proto", fileDescriptor_ef0691a44d1e275c) } var fileDescriptor_ef0691a44d1e275c = []byte{ - // 897 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xdd, 0x72, 0xdb, 0x44, - 0x14, 0xb6, 0x90, 0x6a, 0x59, 0xc7, 0x89, 0x93, 0x6c, 0x20, 0xb8, 0x1e, 0xa0, 0x66, 0x3b, 0x53, - 0xcc, 0xc0, 0x94, 0x12, 0x28, 0x4c, 0x21, 0x5c, 0x34, 0x75, 0xc1, 0x19, 0x60, 0x70, 0xd7, 0xd4, - 0xd7, 0x6c, 0xac, 0x75, 0xd0, 0x54, 0xd1, 0x3a, 0x5a, 0x39, 0xa9, 0x5f, 0x80, 0x37, 0xe0, 0x0d, - 0x78, 0x0b, 0xae, 0x78, 0x33, 0x66, 0x7f, 0x25, 0xb9, 0x31, 0x2d, 0xb9, 0xd3, 0x39, 0xdf, 0x77, - 0xfe, 0xf6, 0xec, 0x9e, 0x23, 0xd8, 0x5a, 0x24, 0x29, 0x17, 0xf4, 0xfe, 0x22, 0xe7, 0x05, 0x47, - 0x4d, 0x2d, 0xe1, 0xdb, 0xe0, 0x4f, 0x87, 0x13, 0x84, 0x20, 0xc8, 0xe8, 0x39, 0xeb, 0x7a, 0x7d, - 0x6f, 0x10, 0x11, 0xf5, 0x8d, 0x77, 0xa1, 0xf3, 0x03, 0x2b, 0xa6, 0xc3, 0x89, 0x20, 0xec, 0x62, - 0xc9, 0x44, 0x81, 0x0f, 0x61, 0xc7, 0x69, 0xc4, 0x82, 0x67, 0x82, 0xa1, 0x3b, 0x10, 0x5c, 0xc6, - 0x42, 0x74, 0xbd, 0xbe, 0x3f, 0x68, 0x1f, 0xb6, 0xef, 0x9b, 0x20, 0xd3, 0xe1, 0x84, 0x28, 0x00, - 0xdf, 0x85, 0x6d, 0x6d, 0x63, 0x9c, 0x5c, 0x1b, 0xea, 0x33, 0x1b, 0xca, 0xf9, 0x7d, 0x1f, 0xfc, - 0xcb, 0x58, 0x28, 0xd2, 0x9a, 0x5b, 0xa9, 0xc7, 0xbf, 0x41, 0x67, 0xcc, 0xc5, 0x6b, 0xdc, 0x4a, - 0xdd, 0x0b, 0xb6, 0x12, 0xdd, 0xb7, 0xfa, 0xde, 0xa0, 0x45, 0xd4, 0x37, 0xba, 0x07, 0x9d, 0x22, - 0xa7, 0xb3, 0x17, 0x4f, 0x5f, 0x26, 0xa2, 0x60, 0xd9, 0x8c, 0x75, 0x7d, 0x85, 0xae, 0x69, 0xf1, - 0x1e, 0xec, 0xb8, 0x08, 0x3a, 0x27, 0x7c, 0x0f, 0x76, 0x87, 0x2c, 0x65, 0x05, 0x7b, 0x4d, 0x35, - 0xfb, 0xb0, 0x57, 0xe1, 0x19, 0xe3, 0x47, 0xb0, 0xf3, 0x6c, 0xc9, 0xf2, 0xd5, 0xf8, 0xd9, 0x4f, - 0xd6, 0xf6, 0x6d, 0xb8, 0x95, 0x64, 0x31, 0x7b, 0x69, 0x8c, 0xb5, 0x80, 0x76, 0xc1, 0x5f, 0x5c, - 0xa4, 0x2a, 0xe7, 0x88, 0xc8, 0x4f, 0x7c, 0xd7, 0x98, 0x4e, 0x4a, 0xd3, 0x5d, 0xf0, 0xc5, 0x45, - 0x6a, 0x0c, 0xe5, 0x27, 0xfe, 0x16, 0xda, 0x93, 0x82, 0x16, 0x4b, 0xf1, 0x34, 0xcf, 0x79, 0x2e, - 0xf3, 0x7a, 0xc2, 0x63, 0x9d, 0xd7, 0x36, 0x51, 0xdf, 0xa8, 0x0b, 0xe1, 0xcf, 0x4c, 0x08, 0x7a, - 0xc6, 0x8c, 0x77, 0x2b, 0xe2, 0xbf, 0x3c, 0x68, 0x13, 0x7e, 0xe5, 0x4e, 0xff, 0x53, 0x08, 0x7f, - 0x67, 0x34, 0x66, 0xb9, 0x6d, 0x2c, 0xb2, 0x1d, 0x78, 0xc2, 0xd3, 0xe5, 0x79, 0x76, 0x92, 0xcd, - 0x39, 0xb1, 0x14, 0xf4, 0x00, 0xc2, 0x99, 0x52, 0xcb, 0x93, 0x96, 0xec, 0x83, 0x3a, 0xdb, 0xba, - 0x25, 0x96, 0x86, 0x1e, 0xd6, 0x92, 0x55, 0x1d, 0x68, 0x1f, 0xee, 0x5b, 0xab, 0x0a, 0x44, 0xaa, - 0x3c, 0xfc, 0x35, 0xf8, 0x84, 0x5f, 0x55, 0xe3, 0x79, 0x6f, 0x14, 0x0f, 0xff, 0xe9, 0xc1, 0xf6, - 0xaf, 0xf4, 0x34, 0x65, 0x37, 0xac, 0xf0, 0x0e, 0x04, 0x39, 0xbf, 0xb2, 0xe5, 0xb9, 0xeb, 0x28, - 0x8f, 0x4c, 0x01, 0x37, 0x2d, 0xe8, 0x08, 0xa0, 0x0c, 0x77, 0xed, 0x15, 0xee, 0x41, 0x2b, 0xa6, - 0x05, 0x2d, 0x56, 0x0b, 0xdb, 0x34, 0x27, 0xe3, 0x3f, 0x7c, 0xe8, 0xd4, 0x2b, 0x46, 0x1f, 0x40, - 0x24, 0x8a, 0x3c, 0xc9, 0xce, 0xa6, 0xd4, 0xdc, 0x8e, 0x51, 0x83, 0x94, 0x2a, 0x89, 0x2f, 0x93, - 0xac, 0xf8, 0xea, 0x4b, 0x89, 0x4b, 0x7f, 0x81, 0xc4, 0x9d, 0x0a, 0xbd, 0x07, 0x2d, 0x07, 0xcb, - 0x22, 0xfc, 0x51, 0x83, 0x38, 0x0d, 0xea, 0x41, 0x78, 0xca, 0x79, 0x2a, 0xc1, 0x40, 0x3e, 0x9a, - 0x51, 0x83, 0x58, 0x85, 0xc2, 0x52, 0x7e, 0x2a, 0xb1, 0x5b, 0x7d, 0x6f, 0xb0, 0xa5, 0x30, 0xad, - 0x40, 0xdf, 0x41, 0x47, 0x87, 0x78, 0x9c, 0xe7, 0x74, 0x25, 0x29, 0xcd, 0xfa, 0x01, 0x3d, 0x2f, - 0xd1, 0x51, 0x83, 0xac, 0x91, 0xa5, 0xb9, 0xae, 0xc0, 0x99, 0x87, 0xeb, 0xe7, 0xeb, 0x50, 0x69, - 0x5e, 0x27, 0xa3, 0x3e, 0xc0, 0x3c, 0xe5, 0xd4, 0x54, 0xd5, 0xea, 0x7b, 0x03, 0x6f, 0xd4, 0x20, - 0x15, 0x1d, 0xfa, 0x1c, 0x20, 0x66, 0xb3, 0xe4, 0x9c, 0xaa, 0xd2, 0x22, 0xe5, 0x7c, 0xc7, 0x3a, - 0x1f, 0x6a, 0x44, 0x9a, 0x94, 0xa4, 0xe3, 0x36, 0x44, 0xfa, 0x72, 0x4d, 0x69, 0x8a, 0x1f, 0x42, - 0x68, 0x58, 0xf2, 0x4d, 0x5f, 0xd2, 0x74, 0xa9, 0x9b, 0xe8, 0x13, 0x2d, 0x48, 0xad, 0x98, 0xd1, - 0x54, 0xb7, 0xd0, 0x27, 0x5a, 0xc0, 0x7f, 0x7b, 0xd0, 0x39, 0xc9, 0xc4, 0x82, 0xcd, 0x8a, 0xff, - 0x1e, 0x09, 0x9f, 0x54, 0x1f, 0x98, 0x4c, 0x6e, 0xcf, 0x26, 0x77, 0x12, 0x8b, 0x5f, 0xf2, 0x1f, - 0xd9, 0x4a, 0x94, 0x6f, 0x0b, 0xc3, 0xd6, 0x3c, 0x49, 0x0b, 0x96, 0x7f, 0x9f, 0xb0, 0x34, 0x16, - 0x5d, 0xbf, 0xef, 0x0f, 0x22, 0x52, 0xd3, 0xc9, 0x30, 0x69, 0x72, 0x9e, 0x14, 0xaa, 0x8d, 0x01, - 0xd1, 0x02, 0x3a, 0x80, 0x26, 0x9f, 0xcf, 0x05, 0x2b, 0x54, 0x07, 0x03, 0x62, 0x24, 0xc9, 0xbe, - 0x90, 0xf3, 0x47, 0x75, 0x2d, 0x22, 0x5a, 0xc0, 0x1f, 0x42, 0xbb, 0xd2, 0x36, 0x79, 0x79, 0x2f, - 0x69, 0xaa, 0x5f, 0x53, 0x40, 0xd4, 0xb7, 0xa4, 0x54, 0x5a, 0x53, 0xa3, 0x44, 0x86, 0x72, 0x06, - 0x91, 0xab, 0x01, 0x7d, 0x04, 0x7e, 0xe2, 0x86, 0xfe, 0x86, 0xcb, 0x21, 0x19, 0xe8, 0xe3, 0xca, - 0x60, 0xdf, 0x78, 0x0f, 0x14, 0xe5, 0xb8, 0x09, 0x81, 0x7c, 0x2c, 0x87, 0xff, 0x04, 0xd0, 0x1c, - 0x2b, 0x1a, 0x3a, 0x82, 0xd0, 0xac, 0x31, 0xe4, 0x26, 0x47, 0x7d, 0xd3, 0xf5, 0xde, 0x7d, 0x45, - 0x6f, 0xc6, 0x78, 0x03, 0x3d, 0x82, 0xa6, 0x56, 0xa2, 0x77, 0xea, 0x24, 0x6b, 0x7b, 0xb0, 0xae, - 0x76, 0xa6, 0x47, 0x10, 0x9a, 0x9d, 0x52, 0x06, 0xae, 0xaf, 0xb1, 0x32, 0xf0, 0xfa, 0xf2, 0x69, - 0xa0, 0x63, 0x88, 0xdc, 0x5a, 0x41, 0xdd, 0xf2, 0x7a, 0xd6, 0x37, 0x52, 0xef, 0xf6, 0x35, 0x48, - 0x25, 0x83, 0x96, 0x5d, 0x25, 0xc8, 0x85, 0x5a, 0x5b, 0x2e, 0xbd, 0xfd, 0xea, 0x7c, 0x73, 0xb6, - 0x0f, 0x3c, 0xf4, 0x18, 0xb6, 0x2d, 0xf7, 0x79, 0x46, 0xf3, 0xd5, 0x66, 0x17, 0xee, 0x68, 0x6a, - 0x53, 0xb7, 0x92, 0xc0, 0xf8, 0x95, 0x04, 0xc6, 0xff, 0x23, 0x81, 0xf1, 0xf5, 0x09, 0x8c, 0xdf, - 0x20, 0x81, 0x6f, 0x20, 0x34, 0x6f, 0xae, 0xec, 0x41, 0xfd, 0x11, 0x6e, 0x0c, 0x7f, 0xda, 0x54, - 0xff, 0x4e, 0x5f, 0xfc, 0x1b, 0x00, 0x00, 0xff, 0xff, 0x1e, 0xff, 0x07, 0x47, 0x4b, 0x09, 0x00, - 0x00, + // 745 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x55, 0xdd, 0x72, 0xd3, 0x3a, + 0x10, 0x8e, 0x6b, 0x37, 0x89, 0x37, 0xfd, 0x3b, 0xea, 0x39, 0x3d, 0x99, 0xce, 0x99, 0x83, 0xeb, + 0x5e, 0x10, 0x06, 0xa6, 0x2d, 0x81, 0xc2, 0x00, 0xe5, 0xa2, 0x2d, 0x30, 0xe9, 0x00, 0x43, 0xaa, + 0xd2, 0x5e, 0x70, 0xa7, 0xc4, 0x4a, 0xea, 0x41, 0xb1, 0x12, 0xcb, 0x69, 0xc9, 0x0b, 0xf0, 0x06, + 0xbc, 0x01, 0x6f, 0xc1, 0x3d, 0xcf, 0xc5, 0x48, 0xb2, 0x1c, 0xbb, 0x10, 0xa6, 0xf4, 0xca, 0xda, + 0xfd, 0xbe, 0xd5, 0xee, 0x6a, 0x7f, 0x0c, 0x0b, 0xc3, 0x90, 0x71, 0x41, 0xb6, 0x86, 0x31, 0x4f, + 0x38, 0x2a, 0x6b, 0xc9, 0x7f, 0x02, 0xcb, 0xc7, 0x63, 0x1a, 0x4f, 0xda, 0xc7, 0x6f, 0x30, 0x1d, + 0x8d, 0xa9, 0x48, 0xd0, 0xdf, 0x30, 0x1f, 0x46, 0x01, 0xfd, 0x54, 0xb7, 0x3c, 0xab, 0xe1, 0x62, + 0x2d, 0xa0, 0x15, 0xb0, 0x87, 0x23, 0x56, 0x9f, 0x53, 0x3a, 0x79, 0xf4, 0x37, 0x53, 0xd3, 0x93, + 0xa9, 0xe9, 0x0a, 0xd8, 0x62, 0xc4, 0x52, 0x43, 0x79, 0xf4, 0x9f, 0x41, 0xed, 0x24, 0x21, 0xc9, + 0x58, 0xbc, 0x8c, 0x63, 0x1e, 0x23, 0x04, 0xce, 0x21, 0x0f, 0xa8, 0x62, 0x2c, 0x62, 0x75, 0x46, + 0x75, 0xa8, 0xbc, 0xa5, 0x42, 0x90, 0x3e, 0x4d, 0x6f, 0x37, 0xa2, 0xff, 0xd5, 0x82, 0x1a, 0xe6, + 0x97, 0x98, 0x8a, 0x21, 0x8f, 0x04, 0x45, 0xf7, 0xa0, 0x72, 0x4e, 0x49, 0x40, 0x63, 0x51, 0xb7, + 0x3c, 0xbb, 0x51, 0x6b, 0xa2, 0xad, 0x34, 0xa9, 0x43, 0xce, 0xc6, 0x83, 0xe8, 0x28, 0xea, 0x71, + 0x6c, 0x28, 0x68, 0x07, 0x2a, 0x5d, 0xa5, 0x16, 0xf5, 0x39, 0xc5, 0x5e, 0x2b, 0xb2, 0xcd, 0xb5, + 0xd8, 0xd0, 0xd0, 0x6e, 0x21, 0xd8, 0xba, 0xed, 0x59, 0x8d, 0x5a, 0x73, 0xd5, 0x58, 0xe5, 0x20, + 0x9c, 0xe7, 0xf9, 0x8f, 0xc1, 0xc6, 0xfc, 0x32, 0xef, 0xcf, 0xba, 0x96, 0x3f, 0xff, 0x8b, 0x05, + 0x8b, 0xef, 0x49, 0x87, 0xd1, 0x1b, 0x66, 0x78, 0x0b, 0x9c, 0x98, 0x5f, 0x9a, 0xf4, 0x6a, 0x86, + 0x2a, 0x9f, 0x4c, 0x01, 0x37, 0x4d, 0x68, 0x0f, 0x60, 0xea, 0x4e, 0xd6, 0x2c, 0x22, 0x03, 0x9a, + 0x56, 0x55, 0x9d, 0xd1, 0x3a, 0x54, 0x03, 0x92, 0x90, 0x64, 0x32, 0x34, 0x45, 0xcb, 0x64, 0xff, + 0xb3, 0x0d, 0x4b, 0xc5, 0x8c, 0xd1, 0xff, 0xe0, 0x8a, 0x24, 0x0e, 0xa3, 0xfe, 0x19, 0x49, 0xbb, + 0xa3, 0x55, 0xc2, 0x53, 0x95, 0xc4, 0xc7, 0x61, 0x94, 0x3c, 0x7a, 0x28, 0x71, 0x79, 0x9f, 0x23, + 0xf1, 0x4c, 0x85, 0xfe, 0x83, 0x6a, 0x06, 0xcb, 0x24, 0xec, 0x56, 0x09, 0x67, 0x1a, 0xb4, 0x0e, + 0x95, 0x0e, 0xe7, 0x4c, 0x82, 0x8e, 0x67, 0x35, 0xaa, 0xad, 0x12, 0x36, 0x0a, 0x85, 0x31, 0xde, + 0x91, 0xd8, 0xbc, 0x67, 0x35, 0x16, 0x14, 0xa6, 0x15, 0xe8, 0x39, 0x2c, 0x69, 0x17, 0xfb, 0x71, + 0x4c, 0x26, 0x92, 0x52, 0x2e, 0x3e, 0xd0, 0xe9, 0x14, 0x6d, 0x95, 0xf0, 0x15, 0xb2, 0x34, 0xd7, + 0x19, 0x64, 0xe6, 0x95, 0xab, 0xef, 0x9b, 0xa1, 0xd2, 0xbc, 0x48, 0x46, 0x1e, 0x40, 0x8f, 0x71, + 0x92, 0x66, 0x55, 0xf5, 0xac, 0x86, 0xd5, 0x2a, 0xe1, 0x9c, 0x0e, 0xdd, 0x07, 0x08, 0x68, 0x37, + 0x1c, 0x10, 0x95, 0x9a, 0xab, 0x2e, 0x5f, 0x36, 0x97, 0xbf, 0xd0, 0x88, 0x34, 0x99, 0x92, 0x0e, + 0x6a, 0xe0, 0xea, 0xe6, 0x3a, 0x23, 0xcc, 0xdf, 0x85, 0x4a, 0xca, 0x92, 0x33, 0x7d, 0x41, 0xd8, + 0x58, 0x17, 0xd1, 0xc6, 0x5a, 0x90, 0x5a, 0xd1, 0x25, 0x4c, 0x97, 0xd0, 0xc6, 0x5a, 0xf0, 0xbf, + 0x59, 0xb0, 0x74, 0x14, 0x89, 0x21, 0xed, 0x26, 0xbf, 0x5f, 0x09, 0x77, 0xf3, 0x03, 0x26, 0x83, + 0xfb, 0xcb, 0x04, 0x77, 0x14, 0x88, 0x77, 0xf1, 0x6b, 0x3a, 0x11, 0xd3, 0xd9, 0xf2, 0x61, 0xa1, + 0x17, 0xb2, 0x84, 0xc6, 0xaf, 0x42, 0xca, 0x02, 0x51, 0xb7, 0x3d, 0xbb, 0xe1, 0xe2, 0x82, 0x4e, + 0xba, 0x61, 0xe1, 0x20, 0x4c, 0x54, 0x19, 0x1d, 0xac, 0x05, 0xb4, 0x06, 0x65, 0xde, 0xeb, 0x09, + 0x9a, 0xa8, 0x0a, 0x3a, 0x38, 0x95, 0x24, 0x7b, 0x24, 0xf7, 0x8f, 0xaa, 0x9a, 0x8b, 0xb5, 0xe0, + 0x6f, 0x40, 0x2d, 0x57, 0x36, 0xd9, 0xbc, 0x17, 0x84, 0xe9, 0x69, 0x72, 0xb0, 0x3a, 0x4b, 0x4a, + 0xae, 0x34, 0x05, 0x8a, 0x9b, 0x52, 0xfa, 0xe0, 0x66, 0x39, 0xa0, 0xdb, 0x60, 0x87, 0x81, 0x50, + 0xb9, 0xcf, 0x6c, 0x0e, 0xc9, 0x40, 0x77, 0xc0, 0xf9, 0x48, 0x27, 0xe6, 0x35, 0x66, 0xf4, 0x81, + 0xa2, 0x1c, 0x94, 0xc1, 0x91, 0xc3, 0xd2, 0xfc, 0x3e, 0x07, 0xe5, 0xb6, 0xa2, 0xa1, 0x3d, 0xa8, + 0x9a, 0x7d, 0x8a, 0xfe, 0x35, 0xb6, 0x57, 0x36, 0xec, 0xfa, 0x6a, 0x7e, 0xc8, 0xd3, 0xf1, 0xf2, + 0x4b, 0x3b, 0x16, 0xda, 0x87, 0x45, 0xc3, 0x3d, 0x8d, 0x48, 0x3c, 0x99, 0x7d, 0xc5, 0x3f, 0x06, + 0x28, 0xac, 0x1e, 0xbf, 0x94, 0x05, 0xd0, 0xfe, 0x29, 0x80, 0xf6, 0x1f, 0x04, 0xd0, 0xfe, 0x75, + 0x00, 0xed, 0x6b, 0x04, 0xf0, 0x14, 0x2a, 0x69, 0xe3, 0xa1, 0x6c, 0x77, 0x16, 0x3b, 0x71, 0xa6, + 0xfb, 0x83, 0xcd, 0x0f, 0x1b, 0xfd, 0x30, 0x39, 0x1f, 0x77, 0xb6, 0xba, 0x7c, 0xb0, 0xad, 0x49, + 0xe6, 0x73, 0xd1, 0xdc, 0x56, 0x7f, 0xbd, 0x4e, 0x59, 0x7d, 0x1e, 0xfc, 0x08, 0x00, 0x00, 0xff, + 0xff, 0x60, 0xce, 0x2e, 0x49, 0x0c, 0x07, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -1265,10 +903,6 @@ const _ = grpc.SupportPackageIsVersion4 // // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. type PilosaClient interface { - GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error) - GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) - PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) - DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error) QuerySQLUnary(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (*TableResponse, error) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Pilosa_QueryPQLClient, error) @@ -1284,42 +918,6 @@ func NewPilosaClient(cc *grpc.ClientConn) PilosaClient { return &pilosaClient{cc} } -func (c *pilosaClient) GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error) { - out := new(GetVDSsResponse) - err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetVDSs", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *pilosaClient) GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) { - out := new(GetVDSResponse) - err := c.cc.Invoke(ctx, "/pilosa.Pilosa/GetVDS", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *pilosaClient) PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) { - out := new(PostVDSResponse) - err := c.cc.Invoke(ctx, "/pilosa.Pilosa/PostVDS", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *pilosaClient) DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) { - out := new(DeleteVDSResponse) - err := c.cc.Invoke(ctx, "/pilosa.Pilosa/DeleteVDS", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *pilosaClient) QuerySQL(ctx context.Context, in *QuerySQLRequest, opts ...grpc.CallOption) (Pilosa_QuerySQLClient, error) { stream, err := c.cc.NewStream(ctx, &_Pilosa_serviceDesc.Streams[0], "/pilosa.Pilosa/QuerySQL", opts...) if err != nil { @@ -1436,10 +1034,6 @@ func (x *pilosaInspectClient) Recv() (*RowResponse, error) { // PilosaServer is the server API for Pilosa service. type PilosaServer interface { - GetVDSs(context.Context, *GetVDSsRequest) (*GetVDSsResponse, error) - GetVDS(context.Context, *GetVDSRequest) (*GetVDSResponse, error) - PostVDS(context.Context, *PostVDSRequest) (*PostVDSResponse, error) - DeleteVDS(context.Context, *DeleteVDSRequest) (*DeleteVDSResponse, error) QuerySQL(*QuerySQLRequest, Pilosa_QuerySQLServer) error QuerySQLUnary(context.Context, *QuerySQLRequest) (*TableResponse, error) QueryPQL(*QueryPQLRequest, Pilosa_QueryPQLServer) error @@ -1451,18 +1045,6 @@ type PilosaServer interface { type UnimplementedPilosaServer struct { } -func (*UnimplementedPilosaServer) GetVDSs(ctx context.Context, req *GetVDSsRequest) (*GetVDSsResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetVDSs not implemented") -} -func (*UnimplementedPilosaServer) GetVDS(ctx context.Context, req *GetVDSRequest) (*GetVDSResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method GetVDS not implemented") -} -func (*UnimplementedPilosaServer) PostVDS(ctx context.Context, req *PostVDSRequest) (*PostVDSResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method PostVDS not implemented") -} -func (*UnimplementedPilosaServer) DeleteVDS(ctx context.Context, req *DeleteVDSRequest) (*DeleteVDSResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method DeleteVDS not implemented") -} func (*UnimplementedPilosaServer) QuerySQL(req *QuerySQLRequest, srv Pilosa_QuerySQLServer) error { return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented") } @@ -1483,78 +1065,6 @@ func RegisterPilosaServer(s *grpc.Server, srv PilosaServer) { s.RegisterService(&_Pilosa_serviceDesc, srv) } -func _Pilosa_GetVDSs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetVDSsRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(PilosaServer).GetVDSs(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/pilosa.Pilosa/GetVDSs", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(PilosaServer).GetVDSs(ctx, req.(*GetVDSsRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Pilosa_GetVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetVDSRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(PilosaServer).GetVDS(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/pilosa.Pilosa/GetVDS", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(PilosaServer).GetVDS(ctx, req.(*GetVDSRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Pilosa_PostVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(PostVDSRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(PilosaServer).PostVDS(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/pilosa.Pilosa/PostVDS", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(PilosaServer).PostVDS(ctx, req.(*PostVDSRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Pilosa_DeleteVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DeleteVDSRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(PilosaServer).DeleteVDS(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/pilosa.Pilosa/DeleteVDS", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(PilosaServer).DeleteVDS(ctx, req.(*DeleteVDSRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _Pilosa_QuerySQL_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(QuerySQLRequest) if err := stream.RecvMsg(m); err != nil { @@ -1658,22 +1168,6 @@ var _Pilosa_serviceDesc = grpc.ServiceDesc{ ServiceName: "pilosa.Pilosa", HandlerType: (*PilosaServer)(nil), Methods: []grpc.MethodDesc{ - { - MethodName: "GetVDSs", - Handler: _Pilosa_GetVDSs_Handler, - }, - { - MethodName: "GetVDS", - Handler: _Pilosa_GetVDS_Handler, - }, - { - MethodName: "PostVDS", - Handler: _Pilosa_PostVDS_Handler, - }, - { - MethodName: "DeleteVDS", - Handler: _Pilosa_DeleteVDS_Handler, - }, { MethodName: "QuerySQLUnary", Handler: _Pilosa_QuerySQLUnary_Handler, diff --git a/proto/pilosa.proto b/proto/pilosa.proto index f51a45502..05bfbd9e4 100644 --- a/proto/pilosa.proto +++ b/proto/pilosa.proto @@ -1,42 +1,9 @@ syntax = "proto3"; package pilosa; -import "public.proto"; +//import "public.proto"; -message VDS { - string name = 1; -} - -message GetVDSsRequest { -} - -message GetVDSsResponse { - repeated VDS vdss = 1; -} - -message GetVDSRequest { - string name = 1; -} - -message GetVDSResponse { - VDS vds = 1; -} - -message PostVDSRequest { - string name = 1; - bool keys = 2; - bool trackExistence = 3; -} - -message PostVDSResponse { -} - -message DeleteVDSRequest { - string name = 1; -} - -message DeleteVDSResponse { -} +option go_package = "github.com/pilosa/pilosa/v2/proto"; message QueryPQLRequest { string index = 1; @@ -117,14 +84,10 @@ message IdsOrKeys { } service Pilosa { - rpc GetVDSs(GetVDSsRequest) returns (GetVDSsResponse) {}; - rpc GetVDS(GetVDSRequest) returns (GetVDSResponse) {}; - rpc PostVDS(PostVDSRequest) returns (PostVDSResponse) {}; - rpc DeleteVDS(DeleteVDSRequest) returns (DeleteVDSResponse) {}; rpc QuerySQL(QuerySQLRequest) returns (stream RowResponse) {}; rpc QuerySQLUnary(QuerySQLRequest) returns (TableResponse) {}; rpc QueryPQL(QueryPQLRequest) returns (stream RowResponse) {}; rpc QueryPQLUnary(QueryPQLRequest) returns (TableResponse) {}; rpc Inspect(InspectRequest) returns (stream RowResponse) {}; - rpc ImportAtomicRecord(stream AtomicRecord) returns (AtomicImportResponse) {}; + //rpc ImportAtomicRecord(stream AtomicRecord) returns (AtomicImportResponse) {}; } diff --git a/proto/vdsm/vdsm.pb.go b/proto/vdsm/vdsm.pb.go new file mode 100644 index 000000000..083d258e8 --- /dev/null +++ b/proto/vdsm/vdsm.pb.go @@ -0,0 +1,1208 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: vdsm/vdsm.proto + +package vdsm + +import ( + context "context" + fmt "fmt" + proto "github.com/golang/protobuf/proto" + proto1 "github.com/pilosa/pilosa/v2/proto" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +// deprecated +type DataSource struct { + Id int64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + Type string `protobuf:"bytes,4,opt,name=type,proto3" json:"type,omitempty"` + Config map[string]string `protobuf:"bytes,5,rep,name=config,proto3" json:"config,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Status string `protobuf:"bytes,6,opt,name=status,proto3" json:"status,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DataSource) Reset() { *m = DataSource{} } +func (m *DataSource) String() string { return proto.CompactTextString(m) } +func (*DataSource) ProtoMessage() {} +func (*DataSource) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{0} +} + +func (m *DataSource) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DataSource.Unmarshal(m, b) +} +func (m *DataSource) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DataSource.Marshal(b, m, deterministic) +} +func (m *DataSource) XXX_Merge(src proto.Message) { + xxx_messageInfo_DataSource.Merge(m, src) +} +func (m *DataSource) XXX_Size() int { + return xxx_messageInfo_DataSource.Size(m) +} +func (m *DataSource) XXX_DiscardUnknown() { + xxx_messageInfo_DataSource.DiscardUnknown(m) +} + +var xxx_messageInfo_DataSource proto.InternalMessageInfo + +func (m *DataSource) GetId() int64 { + if m != nil { + return m.Id + } + return 0 +} + +func (m *DataSource) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *DataSource) GetDescription() string { + if m != nil { + return m.Description + } + return "" +} + +func (m *DataSource) GetType() string { + if m != nil { + return m.Type + } + return "" +} + +func (m *DataSource) GetConfig() map[string]string { + if m != nil { + return m.Config + } + return nil +} + +func (m *DataSource) GetStatus() string { + if m != nil { + return m.Status + } + return "" +} + +type VDS struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Description string `protobuf:"bytes,3,opt,name=description,proto3" json:"description,omitempty"` + PilosaIndex string `protobuf:"bytes,4,opt,name=pilosa_index,json=pilosaIndex,proto3" json:"pilosa_index,omitempty"` + Datasources []*DataSource `protobuf:"bytes,5,rep,name=datasources,proto3" json:"datasources,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *VDS) Reset() { *m = VDS{} } +func (m *VDS) String() string { return proto.CompactTextString(m) } +func (*VDS) ProtoMessage() {} +func (*VDS) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{1} +} + +func (m *VDS) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_VDS.Unmarshal(m, b) +} +func (m *VDS) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_VDS.Marshal(b, m, deterministic) +} +func (m *VDS) XXX_Merge(src proto.Message) { + xxx_messageInfo_VDS.Merge(m, src) +} +func (m *VDS) XXX_Size() int { + return xxx_messageInfo_VDS.Size(m) +} +func (m *VDS) XXX_DiscardUnknown() { + xxx_messageInfo_VDS.DiscardUnknown(m) +} + +var xxx_messageInfo_VDS proto.InternalMessageInfo + +func (m *VDS) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *VDS) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *VDS) GetDescription() string { + if m != nil { + return m.Description + } + return "" +} + +func (m *VDS) GetPilosaIndex() string { + if m != nil { + return m.PilosaIndex + } + return "" +} + +func (m *VDS) GetDatasources() []*DataSource { + if m != nil { + return m.Datasources + } + return nil +} + +type GetVDSsRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVDSsRequest) Reset() { *m = GetVDSsRequest{} } +func (m *GetVDSsRequest) String() string { return proto.CompactTextString(m) } +func (*GetVDSsRequest) ProtoMessage() {} +func (*GetVDSsRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{2} +} + +func (m *GetVDSsRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVDSsRequest.Unmarshal(m, b) +} +func (m *GetVDSsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVDSsRequest.Marshal(b, m, deterministic) +} +func (m *GetVDSsRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVDSsRequest.Merge(m, src) +} +func (m *GetVDSsRequest) XXX_Size() int { + return xxx_messageInfo_GetVDSsRequest.Size(m) +} +func (m *GetVDSsRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetVDSsRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVDSsRequest proto.InternalMessageInfo + +type GetVDSsResponse struct { + Vdss []*VDS `protobuf:"bytes,1,rep,name=vdss,proto3" json:"vdss,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVDSsResponse) Reset() { *m = GetVDSsResponse{} } +func (m *GetVDSsResponse) String() string { return proto.CompactTextString(m) } +func (*GetVDSsResponse) ProtoMessage() {} +func (*GetVDSsResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{3} +} + +func (m *GetVDSsResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVDSsResponse.Unmarshal(m, b) +} +func (m *GetVDSsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVDSsResponse.Marshal(b, m, deterministic) +} +func (m *GetVDSsResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVDSsResponse.Merge(m, src) +} +func (m *GetVDSsResponse) XXX_Size() int { + return xxx_messageInfo_GetVDSsResponse.Size(m) +} +func (m *GetVDSsResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetVDSsResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVDSsResponse proto.InternalMessageInfo + +func (m *GetVDSsResponse) GetVdss() []*VDS { + if m != nil { + return m.Vdss + } + return nil +} + +type GetVDSRequest struct { + // Types that are valid to be assigned to IdOrName: + // *GetVDSRequest_Id + // *GetVDSRequest_Name + IdOrName isGetVDSRequest_IdOrName `protobuf_oneof:"idOrName"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVDSRequest) Reset() { *m = GetVDSRequest{} } +func (m *GetVDSRequest) String() string { return proto.CompactTextString(m) } +func (*GetVDSRequest) ProtoMessage() {} +func (*GetVDSRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{4} +} + +func (m *GetVDSRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVDSRequest.Unmarshal(m, b) +} +func (m *GetVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVDSRequest.Marshal(b, m, deterministic) +} +func (m *GetVDSRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVDSRequest.Merge(m, src) +} +func (m *GetVDSRequest) XXX_Size() int { + return xxx_messageInfo_GetVDSRequest.Size(m) +} +func (m *GetVDSRequest) XXX_DiscardUnknown() { + xxx_messageInfo_GetVDSRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVDSRequest proto.InternalMessageInfo + +type isGetVDSRequest_IdOrName interface { + isGetVDSRequest_IdOrName() +} + +type GetVDSRequest_Id struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3,oneof"` +} + +type GetVDSRequest_Name struct { + Name string `protobuf:"bytes,2,opt,name=name,proto3,oneof"` +} + +func (*GetVDSRequest_Id) isGetVDSRequest_IdOrName() {} + +func (*GetVDSRequest_Name) isGetVDSRequest_IdOrName() {} + +func (m *GetVDSRequest) GetIdOrName() isGetVDSRequest_IdOrName { + if m != nil { + return m.IdOrName + } + return nil +} + +func (m *GetVDSRequest) GetId() string { + if x, ok := m.GetIdOrName().(*GetVDSRequest_Id); ok { + return x.Id + } + return "" +} + +func (m *GetVDSRequest) GetName() string { + if x, ok := m.GetIdOrName().(*GetVDSRequest_Name); ok { + return x.Name + } + return "" +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*GetVDSRequest) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*GetVDSRequest_Id)(nil), + (*GetVDSRequest_Name)(nil), + } +} + +type GetVDSResponse struct { + Vds *VDS `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *GetVDSResponse) Reset() { *m = GetVDSResponse{} } +func (m *GetVDSResponse) String() string { return proto.CompactTextString(m) } +func (*GetVDSResponse) ProtoMessage() {} +func (*GetVDSResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{5} +} + +func (m *GetVDSResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_GetVDSResponse.Unmarshal(m, b) +} +func (m *GetVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_GetVDSResponse.Marshal(b, m, deterministic) +} +func (m *GetVDSResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetVDSResponse.Merge(m, src) +} +func (m *GetVDSResponse) XXX_Size() int { + return xxx_messageInfo_GetVDSResponse.Size(m) +} +func (m *GetVDSResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetVDSResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_GetVDSResponse proto.InternalMessageInfo + +func (m *GetVDSResponse) GetVds() *VDS { + if m != nil { + return m.Vds + } + return nil +} + +type PostVDSRequest struct { + Definition string `protobuf:"bytes,1,opt,name=definition,proto3" json:"definition,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PostVDSRequest) Reset() { *m = PostVDSRequest{} } +func (m *PostVDSRequest) String() string { return proto.CompactTextString(m) } +func (*PostVDSRequest) ProtoMessage() {} +func (*PostVDSRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{6} +} + +func (m *PostVDSRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PostVDSRequest.Unmarshal(m, b) +} +func (m *PostVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PostVDSRequest.Marshal(b, m, deterministic) +} +func (m *PostVDSRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_PostVDSRequest.Merge(m, src) +} +func (m *PostVDSRequest) XXX_Size() int { + return xxx_messageInfo_PostVDSRequest.Size(m) +} +func (m *PostVDSRequest) XXX_DiscardUnknown() { + xxx_messageInfo_PostVDSRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_PostVDSRequest proto.InternalMessageInfo + +func (m *PostVDSRequest) GetDefinition() string { + if m != nil { + return m.Definition + } + return "" +} + +type PostVDSResponse struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Uri string `protobuf:"bytes,2,opt,name=uri,proto3" json:"uri,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PostVDSResponse) Reset() { *m = PostVDSResponse{} } +func (m *PostVDSResponse) String() string { return proto.CompactTextString(m) } +func (*PostVDSResponse) ProtoMessage() {} +func (*PostVDSResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{7} +} + +func (m *PostVDSResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PostVDSResponse.Unmarshal(m, b) +} +func (m *PostVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PostVDSResponse.Marshal(b, m, deterministic) +} +func (m *PostVDSResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_PostVDSResponse.Merge(m, src) +} +func (m *PostVDSResponse) XXX_Size() int { + return xxx_messageInfo_PostVDSResponse.Size(m) +} +func (m *PostVDSResponse) XXX_DiscardUnknown() { + xxx_messageInfo_PostVDSResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_PostVDSResponse proto.InternalMessageInfo + +func (m *PostVDSResponse) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *PostVDSResponse) GetUri() string { + if m != nil { + return m.Uri + } + return "" +} + +type DeleteVDSRequest struct { + // Types that are valid to be assigned to IdOrName: + // *DeleteVDSRequest_Id + // *DeleteVDSRequest_Name + IdOrName isDeleteVDSRequest_IdOrName `protobuf_oneof:"idOrName"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteVDSRequest) Reset() { *m = DeleteVDSRequest{} } +func (m *DeleteVDSRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteVDSRequest) ProtoMessage() {} +func (*DeleteVDSRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{8} +} + +func (m *DeleteVDSRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteVDSRequest.Unmarshal(m, b) +} +func (m *DeleteVDSRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteVDSRequest.Marshal(b, m, deterministic) +} +func (m *DeleteVDSRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVDSRequest.Merge(m, src) +} +func (m *DeleteVDSRequest) XXX_Size() int { + return xxx_messageInfo_DeleteVDSRequest.Size(m) +} +func (m *DeleteVDSRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteVDSRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteVDSRequest proto.InternalMessageInfo + +type isDeleteVDSRequest_IdOrName interface { + isDeleteVDSRequest_IdOrName() +} + +type DeleteVDSRequest_Id struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3,oneof"` +} + +type DeleteVDSRequest_Name struct { + Name string `protobuf:"bytes,2,opt,name=name,proto3,oneof"` +} + +func (*DeleteVDSRequest_Id) isDeleteVDSRequest_IdOrName() {} + +func (*DeleteVDSRequest_Name) isDeleteVDSRequest_IdOrName() {} + +func (m *DeleteVDSRequest) GetIdOrName() isDeleteVDSRequest_IdOrName { + if m != nil { + return m.IdOrName + } + return nil +} + +func (m *DeleteVDSRequest) GetId() string { + if x, ok := m.GetIdOrName().(*DeleteVDSRequest_Id); ok { + return x.Id + } + return "" +} + +func (m *DeleteVDSRequest) GetName() string { + if x, ok := m.GetIdOrName().(*DeleteVDSRequest_Name); ok { + return x.Name + } + return "" +} + +// XXX_OneofWrappers is for the internal use of the proto package. +func (*DeleteVDSRequest) XXX_OneofWrappers() []interface{} { + return []interface{}{ + (*DeleteVDSRequest_Id)(nil), + (*DeleteVDSRequest_Name)(nil), + } +} + +type DeleteVDSResponse struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DeleteVDSResponse) Reset() { *m = DeleteVDSResponse{} } +func (m *DeleteVDSResponse) String() string { return proto.CompactTextString(m) } +func (*DeleteVDSResponse) ProtoMessage() {} +func (*DeleteVDSResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{9} +} + +func (m *DeleteVDSResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteVDSResponse.Unmarshal(m, b) +} +func (m *DeleteVDSResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteVDSResponse.Marshal(b, m, deterministic) +} +func (m *DeleteVDSResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteVDSResponse.Merge(m, src) +} +func (m *DeleteVDSResponse) XXX_Size() int { + return xxx_messageInfo_DeleteVDSResponse.Size(m) +} +func (m *DeleteVDSResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteVDSResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteVDSResponse proto.InternalMessageInfo + +// copied from pilosa.proto to use "vds" instead of "index" +type QueryPQLRequest struct { + Vds string `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` + Pql string `protobuf:"bytes,2,opt,name=pql,proto3" json:"pql,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *QueryPQLRequest) Reset() { *m = QueryPQLRequest{} } +func (m *QueryPQLRequest) String() string { return proto.CompactTextString(m) } +func (*QueryPQLRequest) ProtoMessage() {} +func (*QueryPQLRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{10} +} + +func (m *QueryPQLRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_QueryPQLRequest.Unmarshal(m, b) +} +func (m *QueryPQLRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_QueryPQLRequest.Marshal(b, m, deterministic) +} +func (m *QueryPQLRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_QueryPQLRequest.Merge(m, src) +} +func (m *QueryPQLRequest) XXX_Size() int { + return xxx_messageInfo_QueryPQLRequest.Size(m) +} +func (m *QueryPQLRequest) XXX_DiscardUnknown() { + xxx_messageInfo_QueryPQLRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_QueryPQLRequest proto.InternalMessageInfo + +func (m *QueryPQLRequest) GetVds() string { + if m != nil { + return m.Vds + } + return "" +} + +func (m *QueryPQLRequest) GetPql() string { + if m != nil { + return m.Pql + } + return "" +} + +// copied from pilosa.proto to use "vds" instead of "index" and "records" instead of "columns" +type InspectRequest struct { + Vds string `protobuf:"bytes,1,opt,name=vds,proto3" json:"vds,omitempty"` + Records *proto1.IdsOrKeys `protobuf:"bytes,2,opt,name=records,proto3" json:"records,omitempty"` + FilterFields []string `protobuf:"bytes,3,rep,name=filterFields,proto3" json:"filterFields,omitempty"` + Limit uint64 `protobuf:"varint,4,opt,name=limit,proto3" json:"limit,omitempty"` + Offset uint64 `protobuf:"varint,5,opt,name=offset,proto3" json:"offset,omitempty"` + Query string `protobuf:"bytes,6,opt,name=query,proto3" json:"query,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *InspectRequest) Reset() { *m = InspectRequest{} } +func (m *InspectRequest) String() string { return proto.CompactTextString(m) } +func (*InspectRequest) ProtoMessage() {} +func (*InspectRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_3849c3e549495f50, []int{11} +} + +func (m *InspectRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_InspectRequest.Unmarshal(m, b) +} +func (m *InspectRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_InspectRequest.Marshal(b, m, deterministic) +} +func (m *InspectRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_InspectRequest.Merge(m, src) +} +func (m *InspectRequest) XXX_Size() int { + return xxx_messageInfo_InspectRequest.Size(m) +} +func (m *InspectRequest) XXX_DiscardUnknown() { + xxx_messageInfo_InspectRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_InspectRequest proto.InternalMessageInfo + +func (m *InspectRequest) GetVds() string { + if m != nil { + return m.Vds + } + return "" +} + +func (m *InspectRequest) GetRecords() *proto1.IdsOrKeys { + if m != nil { + return m.Records + } + return nil +} + +func (m *InspectRequest) GetFilterFields() []string { + if m != nil { + return m.FilterFields + } + return nil +} + +func (m *InspectRequest) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *InspectRequest) GetOffset() uint64 { + if m != nil { + return m.Offset + } + return 0 +} + +func (m *InspectRequest) GetQuery() string { + if m != nil { + return m.Query + } + return "" +} + +func init() { + proto.RegisterType((*DataSource)(nil), "vdsm.DataSource") + proto.RegisterMapType((map[string]string)(nil), "vdsm.DataSource.ConfigEntry") + proto.RegisterType((*VDS)(nil), "vdsm.VDS") + proto.RegisterType((*GetVDSsRequest)(nil), "vdsm.GetVDSsRequest") + proto.RegisterType((*GetVDSsResponse)(nil), "vdsm.GetVDSsResponse") + proto.RegisterType((*GetVDSRequest)(nil), "vdsm.GetVDSRequest") + proto.RegisterType((*GetVDSResponse)(nil), "vdsm.GetVDSResponse") + proto.RegisterType((*PostVDSRequest)(nil), "vdsm.PostVDSRequest") + proto.RegisterType((*PostVDSResponse)(nil), "vdsm.PostVDSResponse") + proto.RegisterType((*DeleteVDSRequest)(nil), "vdsm.DeleteVDSRequest") + proto.RegisterType((*DeleteVDSResponse)(nil), "vdsm.DeleteVDSResponse") + proto.RegisterType((*QueryPQLRequest)(nil), "vdsm.QueryPQLRequest") + proto.RegisterType((*InspectRequest)(nil), "vdsm.InspectRequest") +} + +func init() { proto.RegisterFile("vdsm/vdsm.proto", fileDescriptor_3849c3e549495f50) } + +var fileDescriptor_3849c3e549495f50 = []byte{ + // 719 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xcd, 0x6e, 0xda, 0x4c, + 0x14, 0xc5, 0x98, 0x40, 0xb8, 0x24, 0x40, 0x26, 0x24, 0xb1, 0xf8, 0x7e, 0xc4, 0xe7, 0xc5, 0x27, + 0xa4, 0xaa, 0x24, 0x22, 0x8d, 0x44, 0xa3, 0xaa, 0x55, 0x13, 0x9a, 0x26, 0x6a, 0xda, 0x80, 0x69, + 0xb3, 0xe8, 0xa6, 0x72, 0xf0, 0x90, 0x8e, 0x6a, 0x6c, 0xe3, 0x19, 0xd3, 0xf2, 0x3a, 0xdd, 0x77, + 0xdd, 0xc7, 0xea, 0x2b, 0x54, 0xf3, 0x07, 0x86, 0x8a, 0x55, 0x36, 0x30, 0x73, 0xe6, 0xde, 0x7b, + 0xee, 0x99, 0x7b, 0x46, 0x86, 0xca, 0xd4, 0xa3, 0xe3, 0x43, 0xfe, 0xd3, 0x8a, 0xe2, 0x90, 0x85, + 0x28, 0xc7, 0xd7, 0xf5, 0xad, 0x88, 0xf8, 0x21, 0x75, 0x25, 0x66, 0xff, 0x32, 0x00, 0xba, 0x2e, + 0x73, 0x07, 0x61, 0x12, 0x0f, 0x31, 0x2a, 0x43, 0x96, 0x78, 0x96, 0xd1, 0x30, 0x9a, 0xa6, 0x93, + 0x25, 0x1e, 0x42, 0x90, 0x0b, 0xdc, 0x31, 0xb6, 0xb2, 0x0d, 0xa3, 0x59, 0x74, 0xc4, 0x1a, 0x35, + 0xa0, 0xe4, 0x61, 0x3a, 0x8c, 0x49, 0xc4, 0x48, 0x18, 0x58, 0xa6, 0x38, 0x4a, 0x43, 0x3c, 0x8b, + 0xcd, 0x22, 0x6c, 0xe5, 0x64, 0x16, 0x5f, 0xa3, 0x27, 0x90, 0x1f, 0x86, 0xc1, 0x88, 0xdc, 0x5b, + 0x1b, 0x0d, 0xb3, 0x59, 0x6a, 0xff, 0xdd, 0x12, 0x9d, 0x2d, 0xb8, 0x5b, 0xe7, 0xe2, 0xf8, 0x55, + 0xc0, 0xe2, 0x99, 0xa3, 0x62, 0xd1, 0x3e, 0xe4, 0x29, 0x73, 0x59, 0x42, 0xad, 0xbc, 0xa8, 0xa5, + 0x76, 0xf5, 0xa7, 0x50, 0x4a, 0x85, 0xa3, 0x2a, 0x98, 0x5f, 0xf0, 0x4c, 0xf4, 0x5d, 0x74, 0xf8, + 0x12, 0xd5, 0x60, 0x63, 0xea, 0xfa, 0x89, 0xee, 0x5c, 0x6e, 0x4e, 0xb3, 0x1d, 0xc3, 0xfe, 0x6e, + 0x80, 0x79, 0xdb, 0x1d, 0xa4, 0xa4, 0x16, 0x1f, 0x20, 0xf5, 0x3f, 0x50, 0xf7, 0xf9, 0x89, 0x04, + 0x1e, 0xfe, 0xa6, 0x24, 0x97, 0x24, 0x76, 0xc5, 0x21, 0xd4, 0x86, 0x92, 0xe7, 0x32, 0x97, 0x0a, + 0x95, 0x54, 0xc9, 0xaf, 0xae, 0xca, 0x77, 0xd2, 0x41, 0x76, 0x15, 0xca, 0xaf, 0x31, 0xbb, 0xed, + 0x0e, 0xa8, 0x83, 0x27, 0x09, 0xa6, 0xcc, 0x3e, 0x82, 0xca, 0x1c, 0xa1, 0x51, 0x18, 0x50, 0x8c, + 0xfe, 0x01, 0x3e, 0x51, 0x6a, 0x19, 0xa2, 0x62, 0x51, 0x56, 0xbc, 0xed, 0x0e, 0x1c, 0x01, 0xdb, + 0xe7, 0xb0, 0x2d, 0x33, 0x54, 0x09, 0x54, 0x5d, 0x28, 0xbe, 0xcc, 0x08, 0xcd, 0xb5, 0xb4, 0xe6, + 0xcb, 0x8c, 0x54, 0x7d, 0x06, 0xb0, 0x49, 0xbc, 0x9b, 0xf8, 0x9d, 0x3b, 0xc6, 0xf6, 0x63, 0xdd, + 0xc8, 0x9c, 0xf5, 0x2f, 0x30, 0xa7, 0x1e, 0x15, 0x65, 0x96, 0x48, 0x39, 0x6a, 0x1f, 0x41, 0xb9, + 0x17, 0xd2, 0x34, 0xe9, 0xbf, 0x00, 0x1e, 0x1e, 0x91, 0x80, 0x88, 0x1b, 0x94, 0xd7, 0x9d, 0x42, + 0xec, 0x63, 0xa8, 0xcc, 0x33, 0x14, 0xc3, 0xea, 0x64, 0xaa, 0x60, 0x26, 0x31, 0x51, 0x83, 0xe1, + 0x4b, 0xfb, 0x02, 0xaa, 0x5d, 0xec, 0x63, 0x86, 0x1f, 0xa8, 0x6e, 0x17, 0x76, 0x52, 0x75, 0x24, + 0xbd, 0x7d, 0x02, 0x95, 0x7e, 0x82, 0xe3, 0x59, 0xaf, 0x7f, 0xbd, 0xa8, 0x3d, 0xd7, 0x5c, 0x14, + 0x42, 0x39, 0x12, 0x4d, 0x7c, 0xdd, 0x53, 0x34, 0xf1, 0xed, 0x9f, 0x06, 0x94, 0xaf, 0x02, 0x1a, + 0xe1, 0x21, 0x5b, 0x9f, 0xf6, 0x08, 0x0a, 0x31, 0x1e, 0x86, 0xb1, 0x47, 0x45, 0x6a, 0xa9, 0xbd, + 0xd3, 0x52, 0xcf, 0xf1, 0xca, 0xa3, 0x37, 0xf1, 0x1b, 0x3c, 0xa3, 0x8e, 0x8e, 0x40, 0x36, 0x6c, + 0x8d, 0x88, 0xcf, 0x70, 0x7c, 0x41, 0xb0, 0xef, 0x51, 0xcb, 0x6c, 0x98, 0xcd, 0xa2, 0xb3, 0x84, + 0x71, 0x9f, 0xfb, 0x64, 0x4c, 0x98, 0x30, 0x5e, 0xce, 0x91, 0x1b, 0xfe, 0x6c, 0xc2, 0xd1, 0x88, + 0x62, 0x66, 0x6d, 0x08, 0x58, 0xed, 0x78, 0xf4, 0x84, 0x4b, 0x53, 0xaf, 0x49, 0x6e, 0xda, 0x3f, + 0x72, 0xb0, 0xf9, 0x36, 0xf4, 0xf1, 0x30, 0xf1, 0x5d, 0xd4, 0x81, 0x82, 0xf2, 0x19, 0xaa, 0xc9, + 0xe1, 0x2e, 0x1b, 0xb1, 0xbe, 0xb7, 0x82, 0xaa, 0x5b, 0xcb, 0xa0, 0x13, 0xc8, 0x4b, 0x10, 0xed, + 0xa6, 0x43, 0x74, 0x5e, 0x6d, 0x19, 0x9c, 0xa7, 0x75, 0xa0, 0xa0, 0x0c, 0xa0, 0x09, 0x97, 0x1d, + 0xa4, 0x09, 0x57, 0x5c, 0x62, 0x67, 0xd0, 0x73, 0x28, 0xce, 0xa7, 0x87, 0xf6, 0xd5, 0x83, 0x5a, + 0xb1, 0x45, 0xfd, 0xe0, 0x0f, 0x7c, 0x9e, 0xff, 0x0c, 0x36, 0xc5, 0xa0, 0x07, 0xfd, 0x6b, 0x74, + 0xa0, 0xe7, 0xa0, 0x11, 0x9d, 0xbf, 0xab, 0x0f, 0x9c, 0xf0, 0xeb, 0x22, 0xf7, 0xc8, 0x40, 0x2f, + 0x61, 0x5b, 0xc7, 0x7e, 0x08, 0xdc, 0x78, 0xb6, 0xbe, 0xc4, 0x9e, 0x3e, 0x78, 0xef, 0xde, 0xf9, + 0x38, 0xd5, 0xc0, 0xa9, 0x6a, 0xa0, 0xd7, 0xbf, 0x46, 0x4a, 0xe5, 0x8a, 0xf3, 0xd6, 0xd3, 0xbf, + 0x50, 0xf4, 0x3d, 0x4d, 0xbf, 0xa6, 0xc0, 0x5a, 0xf2, 0x0e, 0x14, 0x94, 0x5d, 0xf5, 0xbd, 0x2f, + 0xbb, 0x77, 0x2d, 0xf5, 0x59, 0xf3, 0xe3, 0xff, 0xf7, 0x84, 0x7d, 0x4e, 0xee, 0x5a, 0xc3, 0x70, + 0x7c, 0x28, 0x83, 0xf4, 0xdf, 0xb4, 0x7d, 0x28, 0x3e, 0x2c, 0xe2, 0xbb, 0x73, 0x97, 0x17, 0xeb, + 0xe3, 0xdf, 0x01, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x63, 0x8c, 0x6a, 0x8b, 0x06, 0x00, 0x00, +} + +// Reference imports to suppress errors if they are not otherwise used. +var _ context.Context +var _ grpc.ClientConn + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +const _ = grpc.SupportPackageIsVersion4 + +// MoleculaClient is the client API for Molecula service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. +type MoleculaClient interface { + GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error) + GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) + PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) + DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) + QuerySQL(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) + QuerySQLUnary(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) + QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) + QueryPQLUnary(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) + Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) +} + +type moleculaClient struct { + cc *grpc.ClientConn +} + +func NewMoleculaClient(cc *grpc.ClientConn) MoleculaClient { + return &moleculaClient{cc} +} + +func (c *moleculaClient) GetVDSs(ctx context.Context, in *GetVDSsRequest, opts ...grpc.CallOption) (*GetVDSsResponse, error) { + out := new(GetVDSsResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/GetVDSs", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) GetVDS(ctx context.Context, in *GetVDSRequest, opts ...grpc.CallOption) (*GetVDSResponse, error) { + out := new(GetVDSResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/GetVDS", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) PostVDS(ctx context.Context, in *PostVDSRequest, opts ...grpc.CallOption) (*PostVDSResponse, error) { + out := new(PostVDSResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/PostVDS", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) DeleteVDS(ctx context.Context, in *DeleteVDSRequest, opts ...grpc.CallOption) (*DeleteVDSResponse, error) { + out := new(DeleteVDSResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/DeleteVDS", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) QuerySQL(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (Molecula_QuerySQLClient, error) { + stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[0], "/vdsm.Molecula/QuerySQL", opts...) + if err != nil { + return nil, err + } + x := &moleculaQuerySQLClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Molecula_QuerySQLClient interface { + Recv() (*proto1.RowResponse, error) + grpc.ClientStream +} + +type moleculaQuerySQLClient struct { + grpc.ClientStream +} + +func (x *moleculaQuerySQLClient) Recv() (*proto1.RowResponse, error) { + m := new(proto1.RowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *moleculaClient) QuerySQLUnary(ctx context.Context, in *proto1.QuerySQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) { + out := new(proto1.TableResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/QuerySQLUnary", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) QueryPQL(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (Molecula_QueryPQLClient, error) { + stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[1], "/vdsm.Molecula/QueryPQL", opts...) + if err != nil { + return nil, err + } + x := &moleculaQueryPQLClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Molecula_QueryPQLClient interface { + Recv() (*proto1.RowResponse, error) + grpc.ClientStream +} + +type moleculaQueryPQLClient struct { + grpc.ClientStream +} + +func (x *moleculaQueryPQLClient) Recv() (*proto1.RowResponse, error) { + m := new(proto1.RowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +func (c *moleculaClient) QueryPQLUnary(ctx context.Context, in *QueryPQLRequest, opts ...grpc.CallOption) (*proto1.TableResponse, error) { + out := new(proto1.TableResponse) + err := c.cc.Invoke(ctx, "/vdsm.Molecula/QueryPQLUnary", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *moleculaClient) Inspect(ctx context.Context, in *InspectRequest, opts ...grpc.CallOption) (Molecula_InspectClient, error) { + stream, err := c.cc.NewStream(ctx, &_Molecula_serviceDesc.Streams[2], "/vdsm.Molecula/Inspect", opts...) + if err != nil { + return nil, err + } + x := &moleculaInspectClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type Molecula_InspectClient interface { + Recv() (*proto1.RowResponse, error) + grpc.ClientStream +} + +type moleculaInspectClient struct { + grpc.ClientStream +} + +func (x *moleculaInspectClient) Recv() (*proto1.RowResponse, error) { + m := new(proto1.RowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + +// MoleculaServer is the server API for Molecula service. +type MoleculaServer interface { + GetVDSs(context.Context, *GetVDSsRequest) (*GetVDSsResponse, error) + GetVDS(context.Context, *GetVDSRequest) (*GetVDSResponse, error) + PostVDS(context.Context, *PostVDSRequest) (*PostVDSResponse, error) + DeleteVDS(context.Context, *DeleteVDSRequest) (*DeleteVDSResponse, error) + QuerySQL(*proto1.QuerySQLRequest, Molecula_QuerySQLServer) error + QuerySQLUnary(context.Context, *proto1.QuerySQLRequest) (*proto1.TableResponse, error) + QueryPQL(*QueryPQLRequest, Molecula_QueryPQLServer) error + QueryPQLUnary(context.Context, *QueryPQLRequest) (*proto1.TableResponse, error) + Inspect(*InspectRequest, Molecula_InspectServer) error +} + +// UnimplementedMoleculaServer can be embedded to have forward compatible implementations. +type UnimplementedMoleculaServer struct { +} + +func (*UnimplementedMoleculaServer) GetVDSs(ctx context.Context, req *GetVDSsRequest) (*GetVDSsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetVDSs not implemented") +} +func (*UnimplementedMoleculaServer) GetVDS(ctx context.Context, req *GetVDSRequest) (*GetVDSResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetVDS not implemented") +} +func (*UnimplementedMoleculaServer) PostVDS(ctx context.Context, req *PostVDSRequest) (*PostVDSResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method PostVDS not implemented") +} +func (*UnimplementedMoleculaServer) DeleteVDS(ctx context.Context, req *DeleteVDSRequest) (*DeleteVDSResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method DeleteVDS not implemented") +} +func (*UnimplementedMoleculaServer) QuerySQL(req *proto1.QuerySQLRequest, srv Molecula_QuerySQLServer) error { + return status.Errorf(codes.Unimplemented, "method QuerySQL not implemented") +} +func (*UnimplementedMoleculaServer) QuerySQLUnary(ctx context.Context, req *proto1.QuerySQLRequest) (*proto1.TableResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method QuerySQLUnary not implemented") +} +func (*UnimplementedMoleculaServer) QueryPQL(req *QueryPQLRequest, srv Molecula_QueryPQLServer) error { + return status.Errorf(codes.Unimplemented, "method QueryPQL not implemented") +} +func (*UnimplementedMoleculaServer) QueryPQLUnary(ctx context.Context, req *QueryPQLRequest) (*proto1.TableResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method QueryPQLUnary not implemented") +} +func (*UnimplementedMoleculaServer) Inspect(req *InspectRequest, srv Molecula_InspectServer) error { + return status.Errorf(codes.Unimplemented, "method Inspect not implemented") +} + +func RegisterMoleculaServer(s *grpc.Server, srv MoleculaServer) { + s.RegisterService(&_Molecula_serviceDesc, srv) +} + +func _Molecula_GetVDSs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetVDSsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).GetVDSs(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/GetVDSs", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).GetVDSs(ctx, req.(*GetVDSsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_GetVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetVDSRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).GetVDS(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/GetVDS", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).GetVDS(ctx, req.(*GetVDSRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_PostVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(PostVDSRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).PostVDS(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/PostVDS", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).PostVDS(ctx, req.(*PostVDSRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_DeleteVDS_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteVDSRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).DeleteVDS(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/DeleteVDS", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).DeleteVDS(ctx, req.(*DeleteVDSRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_QuerySQL_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(proto1.QuerySQLRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(MoleculaServer).QuerySQL(m, &moleculaQuerySQLServer{stream}) +} + +type Molecula_QuerySQLServer interface { + Send(*proto1.RowResponse) error + grpc.ServerStream +} + +type moleculaQuerySQLServer struct { + grpc.ServerStream +} + +func (x *moleculaQuerySQLServer) Send(m *proto1.RowResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _Molecula_QuerySQLUnary_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(proto1.QuerySQLRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).QuerySQLUnary(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/QuerySQLUnary", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).QuerySQLUnary(ctx, req.(*proto1.QuerySQLRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_QueryPQL_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(QueryPQLRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(MoleculaServer).QueryPQL(m, &moleculaQueryPQLServer{stream}) +} + +type Molecula_QueryPQLServer interface { + Send(*proto1.RowResponse) error + grpc.ServerStream +} + +type moleculaQueryPQLServer struct { + grpc.ServerStream +} + +func (x *moleculaQueryPQLServer) Send(m *proto1.RowResponse) error { + return x.ServerStream.SendMsg(m) +} + +func _Molecula_QueryPQLUnary_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(QueryPQLRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MoleculaServer).QueryPQLUnary(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/vdsm.Molecula/QueryPQLUnary", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MoleculaServer).QueryPQLUnary(ctx, req.(*QueryPQLRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Molecula_Inspect_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(InspectRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(MoleculaServer).Inspect(m, &moleculaInspectServer{stream}) +} + +type Molecula_InspectServer interface { + Send(*proto1.RowResponse) error + grpc.ServerStream +} + +type moleculaInspectServer struct { + grpc.ServerStream +} + +func (x *moleculaInspectServer) Send(m *proto1.RowResponse) error { + return x.ServerStream.SendMsg(m) +} + +var _Molecula_serviceDesc = grpc.ServiceDesc{ + ServiceName: "vdsm.Molecula", + HandlerType: (*MoleculaServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetVDSs", + Handler: _Molecula_GetVDSs_Handler, + }, + { + MethodName: "GetVDS", + Handler: _Molecula_GetVDS_Handler, + }, + { + MethodName: "PostVDS", + Handler: _Molecula_PostVDS_Handler, + }, + { + MethodName: "DeleteVDS", + Handler: _Molecula_DeleteVDS_Handler, + }, + { + MethodName: "QuerySQLUnary", + Handler: _Molecula_QuerySQLUnary_Handler, + }, + { + MethodName: "QueryPQLUnary", + Handler: _Molecula_QueryPQLUnary_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "QuerySQL", + Handler: _Molecula_QuerySQL_Handler, + ServerStreams: true, + }, + { + StreamName: "QueryPQL", + Handler: _Molecula_QueryPQL_Handler, + ServerStreams: true, + }, + { + StreamName: "Inspect", + Handler: _Molecula_Inspect_Handler, + ServerStreams: true, + }, + }, + Metadata: "vdsm/vdsm.proto", +} diff --git a/proto/vdsm/vdsm.proto b/proto/vdsm/vdsm.proto new file mode 100644 index 000000000..8222a2eb8 --- /dev/null +++ b/proto/vdsm/vdsm.proto @@ -0,0 +1,89 @@ +syntax = "proto3"; +package vdsm; + +option go_package = "github.com/pilosa/pilosa/v2/proto/vdsm"; + +import "pilosa.proto"; + +// deprecated +message DataSource { + int64 id = 1; + string name = 2; + string description = 3; + string type = 4; + map config = 5; + string status = 6; +} + +message VDS { + string id = 1; + string name = 2; + string description = 3; + string pilosa_index = 4; + repeated DataSource datasources = 5; +} + +message GetVDSsRequest { +} + +message GetVDSsResponse { + repeated VDS vdss = 1; +} + +message GetVDSRequest { + oneof idOrName { + string id = 1; + string name = 2; + } +} + +message GetVDSResponse { + VDS vds = 1; +} + +message PostVDSRequest { + string definition = 1; +} + +message PostVDSResponse { + string id = 1; + string uri = 2; +} + +message DeleteVDSRequest { + oneof idOrName { + string id = 1; + string name = 2; + } +} + +message DeleteVDSResponse { +} + +// copied from pilosa.proto to use "vds" instead of "index" +message QueryPQLRequest { + string vds = 1; + string pql = 2; +} + +// copied from pilosa.proto to use "vds" instead of "index" and "records" instead of "columns" +message InspectRequest { + string vds = 1; + pilosa.IdsOrKeys records = 2; + repeated string filterFields = 3; + uint64 limit = 4; + uint64 offset = 5; + string query = 6; +} + +service Molecula { + rpc GetVDSs(GetVDSsRequest) returns (GetVDSsResponse) {}; + rpc GetVDS(GetVDSRequest) returns (GetVDSResponse) {}; + rpc PostVDS(PostVDSRequest) returns (PostVDSResponse) {}; + rpc DeleteVDS(DeleteVDSRequest) returns (DeleteVDSResponse) {}; + rpc QuerySQL(pilosa.QuerySQLRequest) returns (stream pilosa.RowResponse) {}; + rpc QuerySQLUnary(pilosa.QuerySQLRequest) returns (pilosa.TableResponse) {}; + rpc QueryPQL(QueryPQLRequest) returns (stream pilosa.RowResponse) {}; + rpc QueryPQLUnary(QueryPQLRequest) returns (pilosa.TableResponse) {}; + rpc Inspect(InspectRequest) returns (stream pilosa.RowResponse) {}; +} diff --git a/rbf/cursor_test.go b/rbf/cursor_test.go index 40f2756f0..9f134a92e 100644 --- a/rbf/cursor_test.go +++ b/rbf/cursor_test.go @@ -76,8 +76,6 @@ func TestCursor_FirstNext_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { - t.Skip("race detection enabled, skipping") } const n = 10000 @@ -198,8 +196,6 @@ func TestCursor_LastPrev_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { - t.Skip("race detection enabled, skipping") } const n = 10000 @@ -312,8 +308,6 @@ func TestCursor_Union(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { - t.Skip("race detection enabled, skipping") } QuickCheck(t, func(t *testing.T, rand *rand.Rand) { @@ -393,8 +387,6 @@ func TestCursor_Intersect(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { - t.Skip("race detection enabled, skipping") } QuickCheck(t, func(t *testing.T, rand *rand.Rand) { diff --git a/rbf/db.go b/rbf/db.go index 1732eead5..f60b12081 100644 --- a/rbf/db.go +++ b/rbf/db.go @@ -41,12 +41,14 @@ const ( type DB struct { data []byte // mmap data file *os.File // file descriptor - segments []*WALSegment // write-ahead log rootRecords []*RootRecord // cached root records pageMap *immutable.Map // pgno-to-WALID mapping txs map[*Tx]struct{} // active transactions opened bool // true if open + wcache []byte // wal write cache + segments []WALSegment // write-ahead log + mu sync.RWMutex // general mutex rwmu sync.Mutex // mutex for restricting single writer exclmu sync.RWMutex // mutex for locking out everyone but a single writer @@ -71,6 +73,7 @@ func NewDB(path string) *DB { db := &DB{ txs: make(map[*Tx]struct{}), pageMap: immutable.NewMap(&uint32Hasher{}), + wcache: make([]byte, MaxWALSegmentFileSize+PageSize), Path: path, MaxSize: DefaultMaxSize, } @@ -84,12 +87,10 @@ func (db *DB) DataPath() string { // WALPath returns the path to the WAL directory. func (db *DB) WALPath() string { - return filepath.Join(db.Path, "wal") } func CreateDirIfNotExist(path string) { - dir := filepath.Dir(path) if _, err := os.Stat(dir); os.IsNotExist(err) { err = os.MkdirAll(dir, 0755) @@ -99,10 +100,16 @@ func CreateDirIfNotExist(path string) { } } +// TxN returns the number of active transactions. +func (db *DB) TxN() int { + db.mu.RLock() + defer db.mu.RUnlock() + return len(db.txs) +} + // Open opens a database with the file specified in Path. // Creates a new file if one does not already exist. func (db *DB) Open() (err error) { - db.mu.Lock() defer db.mu.Unlock() @@ -143,7 +150,7 @@ func (db *DB) Open() (err error) { // Open write-ahead log & checkpoint to the end since no transactions are open. if err := db.openWALSegments(); err != nil { return fmt.Errorf("wal open: %w", err) - } else if err := db.checkpoint(true); err != nil { + } else if err := db.checkpoint(true, &nopLocker{}); err != nil { return fmt.Errorf("checkpoint: %w", err) } @@ -171,27 +178,66 @@ func (db *DB) openWALSegments() error { } // Truncate everything after the last successful meta page. - if walID, err := db.findLastWALMetaPage(); err != nil { + if walID, err := findLastWALMetaPage(db.segments); err != nil { return err - } else if err := db.truncateWALAfter(walID); err != nil { + } else if db.segments, err = truncateWALAfter(db.segments, walID); err != nil { return err } return nil } -// checkpoint copies pages from WAL segments into the main DB file. This can +// updateWALSegment updates or adds a segment. +func (db *DB) updateWALSegment(s WALSegment) { + segments := make([]WALSegment, len(db.segments), len(db.segments)+1) + copy(segments, db.segments) + + // Find the matching segment using the path. + segment := walSegmentByPath(segments, s.Path) + + // Update existing segment if it already exists. + // Otherwise append segment to the end. + if segment != nil { + *segment = s + } else { + assert(len(segments) == 0 || segments[len(segments)-1].MinWALID < s.MinWALID) + segments = append(segments, s) + } + + // Replace DB segment list. + db.segments = segments +} + +// Checkpoint copies pages from WAL segments into the main DB file. This can // only copy pages that aren't in use by an active transaction. The page map // is rebuilt as well for all WAL pages still in use. // // If exclusive is true, all WAL writes are flushed to disk. -func (db *DB) checkpoint(exclusive bool) error { - if !db.opened { +func (db *DB) Checkpoint() error { + return db.checkpoint(false, &db.mu) +} + +// checkpoint moves WAL segments to the main DB file. +// +// Note that mu should db.mu when called through DB.Checkpoint() but it +// can be &nopLocker if called under lock. The external API will be used +// to periodically checkpoint outside of a transaction and the locking +// must be used only in the beginning (to obtain the segment list) and at +// the end (when removing old segments from the list). If the entire function +// were to obtain a lock then it would block all new read & write transactions. +func (db *DB) checkpoint(exclusive bool, mu sync.Locker) error { + // Obtain a snapshot of WAL segments at the start. + mu.Lock() + opened := db.opened + segments := db.segments + mu.Unlock() + + if !opened { return nil } // Determine last checkpointed WAL ID. - page, err := db.readPage(nil, 0) + page, err := db.readDBPage(0) if err != nil { return err } @@ -206,7 +252,7 @@ func (db *DB) checkpoint(exclusive bool) error { pageMap := immutable.NewMap(&uint32Hasher{}) for { // Determine last page of transaction. - metaWALID, err := db.findNextWALMetaPage(walID) + metaWALID, err := findNextWALMetaPage(segments, walID) if err == io.EOF { break } else if err != nil { @@ -215,9 +261,9 @@ func (db *DB) checkpoint(exclusive bool) error { // Loop over pages in the transaction. for ; walID <= metaWALID; walID++ { - canCheckpoint := exclusive || minActiveWALID == 0 || walID <= minActiveWALID + canCheckpoint := exclusive || minActiveWALID == 0 || walID < minActiveWALID - page, err := db.readWALPage(walID) + page, err := readWALPage(segments, walID) if err != nil { return err } @@ -242,13 +288,13 @@ func (db *DB) checkpoint(exclusive bool) error { // Ensure we actually read the bitmap data in when we checkpoint. // NOTE: The walID variable is incremented above in the pgno check. if isBitmapHeader { - if page, err = db.readWALPage(walID); err != nil { + if page, err = readWALPage(segments, walID); err != nil { return err } } // Write page data into main db file. - if err := db.writePage(pgno, page); err != nil { + if err := db.writeDBPage(pgno, page); err != nil { return err } @@ -259,100 +305,66 @@ func (db *DB) checkpoint(exclusive bool) error { } } + // Ensure WAL pages are fully copied & synced to DB file. + if err := fsync(db.file); err != nil { + return fmt.Errorf("db file sync: %w", err) + } + // Remove WAL segments that have been checkpointed. if maxCheckpointedWALID != 0 { - for len(db.segments) > 0 { - segment := db.segments[0] + for _, segment := range segments { if segment.MaxWALID() > maxCheckpointedWALID { break } - segpath := segment.Path() - if err := segment.Close(); err != nil { - return err - } else if err := os.Remove(segpath); err != nil { + if err := func() error { + mu.Lock() + defer mu.Unlock() + return db.removeWALSegment(segment.Path) + }(); err != nil { return err } - db.segments, db.segments[0] = db.segments[1:], nil } } // Ensure all segments are flushed and there is no remapped pages. if exclusive { + mu.Lock() assert(len(db.segments) == 0) assert(pageMap.Len() == 0) + mu.Unlock() } db.pageMap = pageMap return nil } -// truncateWALAfter removes all pages in the WAL after walID. -func (db *DB) truncateWALAfter(walID int64) error { - for i := len(db.segments) - 1; i >= 0; i-- { - segment := db.segments[i] - if segment.MaxWALID() <= walID { - break - } - - // Drop entire segment if all pages are after WAL ID. - if walID < segment.MinWALID() { +// removeWALSegment closes and deletes the segment with the given path. +// +// The DB's segment list is entirely replaced so that transactions with +// a reference to the old list can continue to use it without a lock. +func (db *DB) removeWALSegment(path string) error { + newSegments := make([]WALSegment, 0, len(db.segments)) + for _, segment := range db.segments { + // Close and remove if path matches. + if segment.Path == path { if err := segment.Close(); err != nil { return err - } else if err := os.Remove(segment.Path()); err != nil { + } else if err := os.Remove(segment.Path); err != nil { return err } - db.segments, db.segments[i] = db.segments[:len(db.segments)-1], nil continue } - // If we only remove some of the WAL pages then truncate and exit - // since segments before this will retain all their pages. - return segment.TruncateAfter(walID) + // Otherwise append to new slice of segments. + newSegments = append(newSegments, segment) } + + // Replace entire slice of segments. + db.segments = newSegments return nil } -func (db *DB) findNextWALMetaPage(walID int64) (metaWALID int64, err error) { - maxWALID := db.maxWALID() - - for ; walID <= maxWALID; walID++ { - // Read page data from WAL and return if it is a meta page. - page, err := db.readWALPage(walID) - if err != nil { - return walID, err - } else if IsMetaPage(page) { - return walID, nil - } - - // Skip over next page if this is a bitmap header. - if IsBitmapHeader(page) { - walID++ - } - } - - return -1, io.EOF -} - -func (db *DB) findLastWALMetaPage() (walID int64, err error) { - if len(db.segments) == 0 { - return 0, nil - } - - var maxMetaWALID int64 - maxWALID := db.maxWALID() - for walID := db.minWALID(); walID <= maxWALID; walID++ { - if page, err := db.readWALPage(walID); err != nil { - return walID, err - } else if IsBitmapHeader(page) { - walID++ // skip next page for bitmap headers - } else if IsMetaPage(page) { - maxMetaWALID = walID // save max meta WAL ID - } - } - return maxMetaWALID, nil -} - // minActiveWALID returns the lowest WAL ID in use by any active transaction. // Returns 0 if no transactions are active. func (db *DB) minActiveWALID() int64 { @@ -365,147 +377,8 @@ func (db *DB) minActiveWALID() int64 { return walID } -// ActiveWALSegment returns the most recent WAL segment. -func (db *DB) ActiveWALSegment() *WALSegment { - db.mu.RLock() - defer db.mu.RUnlock() - return db.activeWALSegment() -} - -func (db *DB) activeWALSegment() *WALSegment { - if len(db.segments) == 0 { - return nil - } - return db.segments[len(db.segments)-1] -} - -// MinWALID returns the lowest WAL ID available in the WAL. -func (db *DB) MinWALID() int64 { - db.mu.RLock() - defer db.mu.RUnlock() - return db.minWALID() -} - -func (db *DB) minWALID() int64 { - if len(db.segments) == 0 { - return 0 - } - return db.segments[0].MinWALID() -} - -// MaxWALID returns the highest WAL ID available in the WAL. -func (db *DB) MaxWALID() int64 { - db.mu.RLock() - defer db.mu.RUnlock() - return db.maxWALID() -} - -func (db *DB) maxWALID() int64 { - - if len(db.segments) == 0 { - return 0 - } - s := db.segments[len(db.segments)-1] - return s.MaxWALID() -} - -// WALPageN returns the number of pages across all segments. -func (db *DB) WALPageN() int64 { - db.mu.RLock() - defer db.mu.RUnlock() - - var n int64 - for _, s := range db.segments { - n += int64(s.PageN()) - } - return n -} - -// SyncWAL flushes the active segment to disk. -func (db *DB) SyncWAL() error { - if s := db.ActiveWALSegment(); s != nil { - return s.Sync() - } - return nil -} - -// readWALPage reads a single page at the given WAL ID. -func (db *DB) readWALPage(walID int64) ([]byte, error) { - // TODO(BBJ): Binary search for segment. - for _, s := range db.segments { - if walID >= s.MinWALID() && walID <= s.MaxWALID() { - return s.ReadWALPage(walID) - } - } - return nil, fmt.Errorf("cannot find segment containing WAL page: %d", walID) -} - -func (db *DB) writeWALPage(page []byte, isMeta bool) (walID int64, err error) { - if err := db.ensureWritableWALSegment(); err != nil { - return 0, err - } - return db.activeWALSegment().WriteWALPage(page, isMeta) -} - -func (db *DB) writeBitmapPage(pgno uint32, page []byte) (walID int64, err error) { - - if err := db.ensureWritableWALSegment(); err != nil { - return 0, err - } - - // Write header page for next bitmap page. - buf := make([]byte, PageSize) - writePageNo(buf[:], pgno) - writeFlags(buf[:], PageTypeBitmapHeader) - // TODO(BBJ): Write checksum. - if _, err := db.activeWALSegment().WriteWALPage(buf, false); err != nil { - return 0, fmt.Errorf("write bitmap header: %w", err) - } - - // Write the bitmap page and return its WALID. - return db.activeWALSegment().WriteWALPage(page, false) -} - -func (db *DB) ensureWritableWALSegment() error { - if s := db.activeWALSegment(); s != nil && s.Size() < MaxWALSegmentFileSize { - return nil - } - return db.addWALSegment() -} - -// addWALSegment appends a new, writable segment and closing an existing segments for write. -func (db *DB) addWALSegment() error { - - // If we have a current active WAL segment then close it and start the - // next segment from the next WAL ID. If there is no existing WAL segments, - // read the last checkpointed WAL ID from the DB and start after that. - var base int64 - if s := db.activeWALSegment(); s != nil { - base = s.MaxWALID() + 1 - if err := s.CloseForWrite(); err != nil { - return err - } - } else { - page, err := db.readPage(db.pageMap, 0) - if err != nil { - return err - } - base = readMetaWALID(page) + 1 - } - - // Create new segment file. - s := NewWALSegment(filepath.Join(db.WALPath(), FormatWALSegmentPath(base))) - if err := s.Open(); err != nil { - return fmt.Errorf("add wal segment: %w", err) - } - db.segments = append(db.segments, s) - - return nil -} - // Close closes the database. func (db *DB) Close() (err error) { - // TODO(bbj): Add wait group to hang until last Tx is complete. // Wait for writer lock. @@ -542,12 +415,12 @@ func (db *DB) Close() (err error) { // closeWALSegments closes the WAL and all its segments. func (db *DB) closeWALSegments() (err error) { - for _, s := range db.segments { if e := s.Close(); e != nil && err == nil { err = e } } + db.segments = nil return err } @@ -613,7 +486,6 @@ func (db *DB) HasData(requireOneHotBit bool) (hasAnyRecords bool, err error) { // Size returns the size of the database & WAL, in bytes. func (db *DB) Size() (int64, error) { - db.mu.RLock() defer db.mu.RUnlock() @@ -621,38 +493,27 @@ func (db *DB) Size() (int64, error) { if err != nil { return 0, err } - return db.walSize() + fi.Size(), nil + return walSize(db.segments) + fi.Size(), nil } // WALSize returns the size of all WAL segments, in bytes. func (db *DB) WALSize() int64 { - db.mu.RLock() defer db.mu.RUnlock() - return db.walSize() -} - -func (db *DB) walSize() int64 { - - var sz int64 - for _, s := range db.segments { - sz += s.Size() - } - return sz + return walSize(db.segments) } // WALSegments returns the WAL segments currently on the DB. -// This should only be used for debugging & testing purposes. -func (db *DB) WALSegments() []*WALSegment { - +func (db *DB) WALSegments() []WALSegment { db.mu.RLock() defer db.mu.RUnlock() - return db.segments + other := make([]WALSegment, len(db.segments)) + copy(other, db.segments) + return other } // init initializes a new database file. func (db *DB) init() error { - if err := db.initMetaPage(); err != nil { return fmt.Errorf("meta: %w", err) } else if err := db.initRootRecordPage(); err != nil { @@ -713,6 +574,10 @@ func (db *DB) BeginWithExclusiveLock() (_ *Tx, err error) { } func (db *DB) begin(writable, exclusive bool) (_ *Tx, err error) { + if exclusive { + assert(writable) // exclusive transactions must be writable + } + if exclusive { db.exclmu.Lock() } else { @@ -749,7 +614,7 @@ func (db *DB) begin(writable, exclusive bool) (_ *Tx, err error) { // Flush all WAL writes to disk before an exclusive writer so that we can // work directly with the on-disk database. if exclusive { - if err := db.checkpoint(true); err != nil { + if err := db.checkpoint(true, &nopLocker{}); err != nil { cleanup() return nil, err } @@ -762,10 +627,21 @@ func (db *DB) begin(writable, exclusive bool) (_ *Tx, err error) { writable: writable, exclusive: exclusive, } + if writable { + tx.wcache = db.wcache[:0] + } + + // Copy list of WAL segments so they can be altered by the tx. + // Add last segment to the list of segments that will be updated/added. + if len(db.segments) != 0 { + tx.segments = make([]WALSegment, len(db.segments)) + copy(tx.segments, db.segments) + tx.updatedSegmentPaths = []string{tx.segments[len(tx.segments)-1].Path} + } // Copy meta page into transaction's buffer. // This page is only written at the end of a dirty transaction. - page, err := db.readPage(db.pageMap, 0) + page, err := db.readMetaPage() if err != nil { tx.Rollback() return nil, err @@ -802,8 +678,8 @@ func (db *DB) removeTx(tx *Tx) error { // Write pages from WAL to DB. // TODO(bbj): Move this to an async goroutine. if tx.writable { - if err := db.checkpoint(false); err != nil { - return err + if err := db.checkpoint(false, &nopLocker{}); err != nil { + return fmt.Errorf("checkpoint: %w", err) } } @@ -824,21 +700,25 @@ func (db *DB) Check() error { return tx.Check() } -// writePage writes a page to the data file. -func (db *DB) writePage(pgno uint32, page []byte) error { +// writeDBPage writes a page to the data file. +func (db *DB) writeDBPage(pgno uint32, page []byte) error { _, err := db.file.WriteAt(page, int64(pgno)*PageSize) return err } -func (db *DB) readPage(pageMap *immutable.Map, pgno uint32) ([]byte, error) { - // Check if page is currently in WAL. - if pageMap != nil { - if walID, ok := pageMap.Get(pgno); ok { - return db.readWALPage(walID.(int64)) - } - } - - // Otherwise read from the data file. +func (db *DB) readDBPage(pgno uint32) ([]byte, error) { offset := int64(pgno) * PageSize return db.data[offset : offset+PageSize], nil } + +func (db *DB) readMetaPage() ([]byte, error) { + if walID, ok := db.pageMap.Get(uint32(0)); ok { + return readWALPage(db.segments, walID.(int64)) + } + return db.readDBPage(0) +} + +type nopLocker struct{} + +func (*nopLocker) Lock() {} +func (*nopLocker) Unlock() {} diff --git a/rbf/db_test.go b/rbf/db_test.go index af0552c6f..a06cd26f6 100644 --- a/rbf/db_test.go +++ b/rbf/db_test.go @@ -112,10 +112,11 @@ func TestDB_Recovery(t *testing.T) { tx1.Rollback() // Close database & truncate WAL to remove commit page & bitmap data page. - segment := db.ActiveWALSegment() + segments := db.WALSegments() + segment := segments[len(segments)-1] if err := db.Close(); err != nil { t.Fatal(err) - } else if err := os.Truncate(segment.Path(), segment.Size()-(2*rbf.PageSize)); err != nil { + } else if err := os.Truncate(segment.Path, segment.Size()-(2*rbf.PageSize)); err != nil { t.Fatal(err) } diff --git a/rbf/os.go b/rbf/os.go index 7b2627f11..a76a550ef 100644 --- a/rbf/os.go +++ b/rbf/os.go @@ -19,4 +19,4 @@ package rbf // DefaultMaxSize is the default mmap size and therefore the maximum allowed // size of the database. The size can be increased by updating the DB.MaxSize // and reopening the database. This setting mainly affects virtual space usage. -const DefaultMaxSize = 100 * (1 << 30) // 100GB +const DefaultMaxSize = 4 * (1 << 30) diff --git a/rbf/rbf.go b/rbf/rbf.go index f152e3499..b504e4fbc 100644 --- a/rbf/rbf.go +++ b/rbf/rbf.go @@ -91,6 +91,11 @@ var ( // Debug is just a temporary flag used for debugging. var Debug bool +// Testing constants. +const ( + SyncEnabled = true +) + // Magic32 returns the magic bytes as a big endian encoded uint32. func Magic32() uint32 { return binary.BigEndian.Uint32([]byte(Magic)) @@ -648,3 +653,26 @@ func RowValues(b []uint64) []uint64 { // _, file, line, _ := runtime.Caller(skip + 1) // return fmt.Sprintf("%s:%d", file, line) // } + +// truncate truncates the file at path to sz bytes. File must exist. +func truncate(path string, sz int64) error { + f, err := os.OpenFile(path, os.O_WRONLY, 0666) + if err != nil { + return fmt.Errorf("open file: %w", err) + } + defer f.Close() + + if err := f.Truncate(sz); err != nil { + return fmt.Errorf("truncate: %w", err) + } else if err := fsync(f); err != nil { + return fmt.Errorf("sync: %w", err) + } + return f.Close() +} + +func fsync(f *os.File) error { + if !SyncEnabled { + return nil + } + return f.Sync() +} diff --git a/rbf/rbf_norace.go b/rbf/rbf_norace.go deleted file mode 100644 index 15667ee93..000000000 --- a/rbf/rbf_norace.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build !race - -package rbf - -// RaceEnabled is true if the -race flag is enabled. -const RaceEnabled = false diff --git a/rbf/rbf_race.go b/rbf/rbf_race.go deleted file mode 100644 index e178a736b..000000000 --- a/rbf/rbf_race.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 Pilosa Corp. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// +build race - -package rbf - -// RaceEnabled is true if the -race flag is enabled. -const RaceEnabled = true diff --git a/rbf/rbf_test.go b/rbf/rbf_test.go index 8b4768cd5..e4ea450e2 100644 --- a/rbf/rbf_test.go +++ b/rbf/rbf_test.go @@ -86,6 +86,8 @@ func MustCloseDB(tb testing.TB, db *rbf.DB) { tb.Helper() if err := db.Check(); err != nil && err != rbf.ErrClosed { tb.Fatal(err) + } else if n := db.TxN(); n != 0 { + tb.Fatalf("db still has %d active transactions; must closed before closing db", n) } else if err := db.Close(); err != nil && err != rbf.ErrClosed { tb.Fatal(err) } else if err := os.RemoveAll(db.Path); err != nil { diff --git a/rbf/tx.go b/rbf/tx.go index 5be536221..c2647b20f 100644 --- a/rbf/tx.go +++ b/rbf/tx.go @@ -17,9 +17,9 @@ import ( "fmt" "io" "math" + "os" + "path/filepath" "sort" - - //"strconv" "strings" "sync" @@ -33,15 +33,19 @@ var _ = txkey.ToString // Tx represents a transaction. type Tx struct { - mu sync.RWMutex - db *DB // parent db - meta [PageSize]byte // copy of current meta page - walID int64 // max WAL ID at start of tx - rootRecords []*RootRecord // read-only cache of root records - pageMap *immutable.Map // mapping of database pages to WAL IDs - writable bool // if true, tx can write - exclusive bool // if true, tx writes directly to db file (no wal) - dirty bool // if true, changes have been made + mu sync.RWMutex + db *DB // parent db + segments []WALSegment // copy of WAL segments + updatedSegmentPaths []string // updated or added segment paths + meta [PageSize]byte // copy of current meta page + walID int64 // max WAL ID at start of tx + rootRecords []*RootRecord // read-only cache of root records + pageMap *immutable.Map // mapping of database pages to WAL IDs + writable bool // if true, tx can write + exclusive bool // if true, tx writes directly to db file (no wal) + dirty bool // if true, changes have been made + + wcache []byte // write cache // If Rollback() has already completed, don't do it again. // Note db == nil means that commit has already been done. @@ -76,7 +80,7 @@ func (tx *Tx) Commit() error { if tx.dirty { if err := tx.writeMetaPage(MetaPageFlagCommit); err != nil { return err - } else if err := tx.db.SyncWAL(); err != nil { + } else if err := tx.flushWALWriter(); err != nil { return err } @@ -89,6 +93,11 @@ func (tx *Tx) Commit() error { tx.db.mu.Lock() tx.db.rootRecords = tx.rootRecords tx.db.pageMap = tx.pageMap + for _, path := range tx.updatedSegmentPaths { + segment := walSegmentByPath(tx.segments, path) + assert(segment != nil) + tx.db.updateWALSegment(*segment) + } tx.db.mu.Unlock() } @@ -112,11 +121,12 @@ func (tx *Tx) Rollback() { // TODO(bbj): Invalidate DB if rollback fails. Possibly attempt reopen? - // Remove all WAL pages that have been written by this transaction. if tx.dirty { - if err := tx.db.truncateWALAfter(tx.walID); err != nil { - panic(err) + if _, err := truncateWALAfter(tx.segments, tx.walID); err != nil { + panicOn(err) } + tx.segments = nil + tx.updatedSegmentPaths = nil } // Disconnect transaction from DB. @@ -915,11 +925,28 @@ func (tx *Tx) readPage(pgno uint32) ([]byte, error) { return tx.meta[:], nil } + // Verify page number requested is within current size of database. pageN := readMetaPageN(tx.meta[:]) if pgno > pageN { return nil, fmt.Errorf("rbf: page read out of bounds: pgno=%d max=%d", pgno, pageN) } - return tx.db.readPage(tx.pageMap, pgno) + + // Check if page is remapped. + if walID, ok := tx.pageMap.Get(pgno); ok { + walID64 := walID.(int64) + + // Read from write cache if not yet flushed to disk. + maxWALID := activeWALSegment(tx.segments).MaxWALID() + if walID64 > maxWALID { + offset := (walID64 - maxWALID - 1) * PageSize + return tx.wcache[offset : offset+PageSize], nil + } + + // Otherwise return remapped page from WAL segment. + return readWALPage(tx.segments, walID64) + } + + return tx.db.readDBPage(pgno) } func (tx *Tx) writePage(page []byte) error { @@ -928,11 +955,11 @@ func (tx *Tx) writePage(page []byte) error { // If we are running in exclusive mode, directly write page to database. if tx.exclusive { - return tx.db.writePage(readPageNo(page), page) + return tx.db.writeDBPage(readPageNo(page), page) } // Write page to WAL and obtain position in WAL. - walID, err := tx.db.writeWALPage(page, false) + walID, err := tx.writeWALPage(page, false) if err != nil { return err } @@ -948,11 +975,11 @@ func (tx *Tx) writeBitmapPage(pgno uint32, page []byte) error { // If we are running in exclusive mode, directly write page to database. if tx.exclusive { - return tx.db.writePage(pgno, page) + return tx.db.writeDBPage(pgno, page) } // Write bitmap to WAL and obtain WAL position of the actual page data (not the prefix page). - walID, err := tx.db.writeBitmapPage(pgno, page) + walID, err := tx.writeBitmapWALPage(pgno, page) if err != nil { return err } @@ -968,11 +995,11 @@ func (tx *Tx) writeMetaPage(flag uint32) error { // If we are running in exclusive mode, directly write page to database. if tx.exclusive { - return tx.db.writePage(0, tx.meta[:]) + return tx.db.writeDBPage(0, tx.meta[:]) } // Write page to WAL and obtain position in WAL. - walID, err := tx.db.writeWALPage(tx.meta[:], true) + walID, err := tx.writeWALPage(tx.meta[:], true) if err != nil { return err } @@ -1562,3 +1589,112 @@ func (tx *Tx) ImportRoaringBits(name string, itr roaring.RoaringIterator, clear } return } + +func (tx *Tx) flushWALWriter() error { + // Ignore if we have no data in the write cache. + if len(tx.wcache) == 0 { + return nil + } + + // Determine active WAL segment. + assert(len(tx.segments) != 0) + segment := &tx.segments[len(tx.segments)-1] + + // Open write handle to active segment. + w, err := os.OpenFile(segment.Path, os.O_WRONLY, 0666) + if err != nil { + return fmt.Errorf("open wal segment write handle: %w", err) + } + defer w.Close() + + // Flush cache to writer. + if _, err := w.WriteAt(tx.wcache, int64(segment.PageN)*PageSize); err != nil { + return fmt.Errorf("write wal segment: %w", err) + } else if err := fsync(w); err != nil { + return fmt.Errorf("sync wal segment: %w", err) + } else if err := w.Close(); err != nil { + return fmt.Errorf("close wal segment: %w", err) + } + + // Increase the size of the last WAL segment & clear cache. + assert(len(tx.wcache)%PageSize == 0) + segment.PageN += len(tx.wcache) / PageSize + tx.wcache = tx.wcache[:0] + + return nil +} + +func (tx *Tx) writeWALPage(page []byte, isMeta bool) (walID int64, err error) { + if err := tx.ensureWritableWALSegment(); err != nil { + return 0, err + } + + // Determine next WAL ID from cached meta page. + walID = readMetaWALID(tx.meta[:]) + 1 + + // Update WAL ID on cached meta page. + writeMetaWALID(tx.meta[:], walID) + + // Append write to write buffer. + tx.wcache = append(tx.wcache, page...) + + return walID, nil +} + +func (tx *Tx) writeBitmapWALPage(pgno uint32, page []byte) (walID int64, err error) { + if err := tx.ensureWritableWALSegment(); err != nil { + return 0, err + } + + // Write header page for next bitmap page. + buf := make([]byte, PageSize) + writePageNo(buf[:], pgno) + writeFlags(buf[:], PageTypeBitmapHeader) + // TODO(BBJ): Write checksum. + if _, err := tx.writeWALPage(buf, false); err != nil { + return 0, fmt.Errorf("write bitmap header: %w", err) + } + + // Write the bitmap page and return its WALID. + return tx.writeWALPage(page, false) +} + +func (tx *Tx) ensureWritableWALSegment() error { + // Ignore if we still have space in the write cache. + writeCacheSize := int64(len(tx.wcache)) + if len(tx.segments) != 0 && activeWALSegment(tx.segments).Size()+writeCacheSize < MaxWALSegmentFileSize { + return nil + } + + // Flush write cache out to file before adding new segment. + if err := tx.flushWALWriter(); err != nil { + return err + } + + // If we have a current active WAL segment then close it and start the + // next segment from the next WAL ID. If there is no existing WAL segments, + // read the last checkpointed WAL ID from the DB and start after that. + var base int64 + if len(tx.segments) != 0 { + base = activeWALSegment(tx.segments).MaxWALID() + 1 + } else { + page, err := tx.readPage(0) + if err != nil { + return err + } + base = readMetaWALID(page) + 1 + } + + // Create new segment file. + s := NewWALSegment(filepath.Join(tx.db.WALPath(), FormatWALSegmentPath(base))) + if err := s.Open(); err != nil { + return fmt.Errorf("add wal segment: %w", err) + } + + // Track all segments that need to be added back to DB. + // The DB can remove segments in the background so we don't want to replace. + tx.segments = append(tx.segments, s) + tx.updatedSegmentPaths = append(tx.updatedSegmentPaths, s.Path) + + return nil +} diff --git a/rbf/tx_test.go b/rbf/tx_test.go index defd0ac23..2d5505a4a 100644 --- a/rbf/tx_test.go +++ b/rbf/tx_test.go @@ -218,8 +218,6 @@ func TestTx_Add_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { - t.Skip("race detection enabled, skipping") } QuickCheck(t, func(t *testing.T, rand *rand.Rand) { @@ -257,8 +255,6 @@ func TestTx_AddRemove_Quick(t *testing.T) { t.Skip("-short enabled, skipping") } else if is32Bit() { t.Skip("32-bit build, skipping quick check tests") - } else if rbf.RaceEnabled { - t.Skip("race detection enabled, skipping") } QuickCheck(t, func(t *testing.T, rand *rand.Rand) { diff --git a/rbf/wal.go b/rbf/wal.go index 7a0518d96..14e64a107 100644 --- a/rbf/wal.go +++ b/rbf/wal.go @@ -16,9 +16,9 @@ package rbf import ( "fmt" + "io" "os" "path/filepath" - "sync" "syscall" "github.com/pilosa/pilosa/v2/syswrap" @@ -26,66 +26,40 @@ import ( // WALSegment represents a single file in the WAL. type WALSegment struct { - mu sync.RWMutex - minWALID int64 // base WALID; calculated from path - path string // path to file - w *os.File // write handle - data []byte // read-only mmap data - writeCache []byte // write buffer - pageN int // number of written pages + Path string // path to file + MinWALID int64 // base WALID; calculated from path + PageN int // number of written pages + + data []byte // read-only mmap data } // NewWALSegment returns a new instance of WALSegment for a given path. -func NewWALSegment(path string) *WALSegment { - return &WALSegment{ - path: path, +func NewWALSegment(path string) WALSegment { + return WALSegment{ + Path: path, } } -// Path returns the path the segment was initialized with. -func (s *WALSegment) Path() string { return s.path } - -// MinWALID returns the initial WAL ID of the segment. Only available after Open(). -func (s *WALSegment) MinWALID() int64 { - s.mu.RLock() - defer s.mu.RUnlock() - return s.minWALID -} - // MaxWALID returns the maximum WAL ID of the segment. Only available after Open(). -func (s *WALSegment) MaxWALID() int64 { - s.mu.RLock() - defer s.mu.RUnlock() - return s.minWALID + int64(s.pageN) - 1 -} - -// PageN returns the number of pages in the segment. -func (s *WALSegment) PageN() int { - s.mu.RLock() - defer s.mu.RUnlock() - return s.pageN +func (s WALSegment) MaxWALID() int64 { + return s.MinWALID + int64(s.PageN) - 1 } // Size returns the current size of the segment, in bytes. -func (s *WALSegment) Size() int64 { - s.mu.RLock() - defer s.mu.RUnlock() - return int64(s.pageN) * PageSize +func (s WALSegment) Size() int64 { + return int64(s.PageN) * PageSize } func (s *WALSegment) Open() (err error) { - s.mu.Lock() - defer s.mu.Unlock() - // Extract base WAL ID and validate path. - if s.minWALID, err = ParseWALSegmentPath(s.path); err != nil { + if s.MinWALID, err = ParseWALSegmentPath(s.Path); err != nil { return err } // Determine file size & create if necessary. var sz int64 - if fi, err := os.Stat(s.path); os.IsNotExist(err) { - if f, err := os.OpenFile(s.path, os.O_RDWR|os.O_CREATE, 0666); err != nil { + if fi, err := os.Stat(s.Path); os.IsNotExist(err) { + if f, err := os.OpenFile(s.Path, os.O_RDWR|os.O_CREATE, 0666); err != nil { return fmt.Errorf("touch wal segment file: %w", err) } else if err := f.Close(); err != nil { return fmt.Errorf("close touched wal segment file: %w", err) @@ -97,11 +71,11 @@ func (s *WALSegment) Open() (err error) { } // Determine page count & truncate if a partial page is written. - s.pageN = int(sz / PageSize) + s.PageN = int(sz / PageSize) if sz%PageSize != 0 { - sz = int64(s.pageN * PageSize) - if err := os.Truncate(s.path, sz); err != nil { - return fmt.Errorf("truncate wal segment file: %w", err) + sz = int64(s.PageN * PageSize) + if err := truncate(s.Path, sz); err != nil { + return fmt.Errorf("truncate wal file: %w", err) } } @@ -113,7 +87,7 @@ func (s *WALSegment) Open() (err error) { } // Open file as a read-only memory map. - if f, err := os.OpenFile(s.path, os.O_RDONLY, 0666); err != nil { + if f, err := os.OpenFile(s.Path, os.O_RDONLY, 0666); err != nil { return fmt.Errorf("open wal segment file: %w", err) } else if s.data, err = syswrap.Mmap(int(f.Fd()), 0, int(mmapSize), syscall.PROT_READ, syscall.MAP_SHARED); err != nil { f.Close() @@ -127,12 +101,6 @@ func (s *WALSegment) Open() (err error) { // Close closes the write handle and the read-only mmap. func (s *WALSegment) Close() error { - s.mu.Lock() - defer s.mu.Unlock() - - if err := s.closeForWrite(); err != nil { - return err - } if s.data != nil { if err := syswrap.Munmap(s.data); err != nil { return err @@ -142,144 +110,148 @@ func (s *WALSegment) Close() error { return nil } -// CloseForWrite closes the write handle, if initialized. -func (s *WALSegment) CloseForWrite() error { - s.mu.Lock() - defer s.mu.Unlock() - return s.closeForWrite() -} - -func (s *WALSegment) closeForWrite() error { - // Ensure write buffer is flushed out. - if err := s.sync(); err != nil { - return err - } - s.writeCache = nil - - // Close underlying file writer. - if s.w != nil { - if err := s.w.Close(); err != nil { - return err - } - s.w = nil - } - return nil -} - // ReadWALPage reads a single page at the given WAL ID. func (s *WALSegment) ReadWALPage(walID int64) ([]byte, error) { - s.mu.RLock() - defer s.mu.RUnlock() - // Ensure requested ID is contained in this file. - if walID < s.minWALID || walID > s.minWALID+int64(s.pageN) { - return nil, fmt.Errorf("wal segment page read out of range: id=%d base=%d pageN=%d", walID, s.minWALID, s.pageN) + if walID < s.MinWALID || walID > s.MinWALID+int64(s.PageN) { + return nil, fmt.Errorf("wal segment page read out of range: id=%d base=%d pageN=%d", walID, s.MinWALID, s.PageN) } - offset := (walID - s.minWALID) * PageSize - - // If offset is within write buffer, return from write buffer. - writeBufferOffset := int64((s.pageN * PageSize) - len(s.writeCache)) - if offset >= writeBufferOffset { - buf := s.writeCache[offset-writeBufferOffset:] - return buf[:PageSize:PageSize], nil - } - - // Otherwise return from on-disk mmap. + offset := (walID - s.MinWALID) * PageSize return s.data[offset : offset+PageSize], nil } -// WriteWALPage writes a single page to the WAL segment and returns its WAL identifier. -func (s *WALSegment) WriteWALPage(page []byte, isMeta bool) (walID int64, err error) { - assert(len(page) == PageSize) // invalid page size - - s.mu.Lock() - defer s.mu.Unlock() - - // Initialize write file handle if not yet initialized. - if s.w == nil { - if s.w, err = os.OpenFile(s.path, os.O_WRONLY, 0666); err != nil { - return 0, fmt.Errorf("open wal segment write handle: %w", err) +func walSegmentByPath(segments []WALSegment, path string) *WALSegment { + for i := range segments { + if segments[i].Path == path { + return &segments[i] } } - - // Determine current WAL position. - walID = s.minWALID + int64(s.pageN) - - // Write WAL ID if this is a meta page. - if isMeta { - writeMetaWALID(page, walID) - // TODO: Write meta page checksum - } - - // Append write to write buffer & increment page count. - if s.writeCache == nil { - s.writeCache = make([]byte, 0, MaxWALSegmentFileSize+PageSize) - } - s.writeCache = append(s.writeCache, page...) - s.pageN++ - - return walID, nil -} - -// TruncateAfter removes all pages after a given WAL ID. -func (s *WALSegment) TruncateAfter(walID int64) error { - s.mu.Lock() - defer s.mu.Unlock() - - // Ensure this is a partial truncation. Full truncation of a segment - // should be performed by the DB since it needs to remove the segment. - assert(walID > s.minWALID) - - // Update to new page size. - newPageN := int((walID - s.minWALID) + 1) // new page count of segment - truncPageN := s.pageN - newPageN // number of pages removed - s.pageN = newPageN - - // Check to see if we are only truncating from the write cache. - writeCachePageN := len(s.writeCache) / PageSize - if truncPageN <= int(writeCachePageN) { - s.writeCache = s.writeCache[:(writeCachePageN-truncPageN)*PageSize] - return nil - } - - // Clear write cache. - s.writeCache = s.writeCache[:0] - - // Remove on disk pages. - return os.Truncate(s.path, int64(s.pageN)*PageSize) -} - -// Flush flushes the write buffer to the OS cache. -func (s *WALSegment) Flush() error { - s.mu.Lock() - defer s.mu.Unlock() - return s.flush() -} - -func (s *WALSegment) flush() error { - if _, err := s.w.WriteAt(s.writeCache, int64((s.pageN*PageSize)-len(s.writeCache))); err != nil { - return fmt.Errorf("wal segment write: %w", err) - } - s.writeCache = s.writeCache[:0] return nil } -// Sync flushes the write buffer and invokes a file sync to flush data to disk. -func (s *WALSegment) Sync() error { - s.mu.Lock() - defer s.mu.Unlock() - return s.sync() +func activeWALSegment(segments []WALSegment) WALSegment { + if len(segments) == 0 { + return WALSegment{} + } + return segments[len(segments)-1] } -func (s *WALSegment) sync() error { - if s.w == nil { - return nil +func minWALID(segments []WALSegment) int64 { + if len(segments) == 0 { + return 0 } - if err := s.flush(); err != nil { - return err + return segments[0].MinWALID +} + +func maxWALID(segments []WALSegment) int64 { + if len(segments) == 0 { + return 0 + } + s := segments[len(segments)-1] + return s.MaxWALID() +} + +func walSize(segments []WALSegment) int64 { + var sz int64 + for _, s := range segments { + sz += s.Size() + } + return sz +} + +// readWALPage reads a single page at the given WAL ID. +func readWALPage(segments []WALSegment, walID int64) ([]byte, error) { + // TODO(BBJ): Binary search for segment. + for _, s := range segments { + if walID >= s.MinWALID && walID <= s.MaxWALID() { + return s.ReadWALPage(walID) + } + } + return nil, fmt.Errorf("cannot find segment containing WAL page: %d", walID) +} + +func findNextWALMetaPage(segments []WALSegment, walID int64) (metaWALID int64, err error) { + maxWALID := maxWALID(segments) + + for ; walID <= maxWALID; walID++ { + // Read page data from WAL and return if it is a meta page. + page, err := readWALPage(segments, walID) + if err != nil { + return walID, err + } else if IsMetaPage(page) { + return walID, nil + } + + // Skip over next page if this is a bitmap header. + if IsBitmapHeader(page) { + walID++ + } + } + + return -1, io.EOF +} + +func findLastWALMetaPage(segments []WALSegment) (walID int64, err error) { + if len(segments) == 0 { + return 0, nil + } + + var maxMetaWALID int64 + maxWALID := maxWALID(segments) + for walID := minWALID(segments); walID <= maxWALID; walID++ { + if page, err := readWALPage(segments, walID); err != nil { + return walID, err + } else if IsBitmapHeader(page) { + walID++ // skip next page for bitmap headers + } else if IsMetaPage(page) { + maxMetaWALID = walID // save max meta WAL ID + } + } + return maxMetaWALID, nil +} + +// truncateWALAfter removes all pages in the WAL after walID. +func truncateWALAfter(segments []WALSegment, walID int64) ([]WALSegment, error) { + var newSegments []WALSegment + + for i := range segments { + segment := &segments[i] + + // Append entire segment if WAL range entirely before target WAL ID. + if walID > segment.MaxWALID() { + newSegments = append(newSegments, *segment) + continue + } + + // If we only remove some of the WAL pages then truncate and append. + if segment.MinWALID < walID { + newSegment := *segment + newSegment.PageN = int((walID - newSegment.MinWALID) + 1) + + if err := truncate(newSegment.Path, int64(newSegment.PageN)*PageSize); err != nil { + return segments, err + } + newSegments = append(newSegments, newSegment) + continue + } + + // Drop entire segment if all pages are after WAL ID. + if err := segment.Close(); err != nil { + return segments, err + } else if err := os.Remove(segment.Path); err != nil { + return segments, err + } + } + + return newSegments, nil +} + +func DumpWALSegments(segments []WALSegment) { + fmt.Printf("WAL (%d segments)\n", len(segments)) + for i, s := range segments { + fmt.Printf("[%d] WALIDs=(%d-%d) PageN=%d\n", i, s.MinWALID, s.MaxWALID(), s.PageN) } - return s.w.Sync() } // FormatWALSegmentPath returns a path for a WAL segment using a WAL ID. diff --git a/rbf/wal_test.go b/rbf/wal_test.go index b41e7e792..59463d435 100644 --- a/rbf/wal_test.go +++ b/rbf/wal_test.go @@ -15,10 +15,10 @@ package rbf_test import ( - "bytes" - "encoding/hex" + // "bytes" + // "encoding/hex" "io/ioutil" - "math/rand" + // "math/rand" "os" "path/filepath" "testing" @@ -30,9 +30,9 @@ func TestWALSegment_Open(t *testing.T) { t.Run("OK", func(t *testing.T) { s := MustOpenWALSegment(t, 10) defer MustCloseWALSegment(t, s) - if got, want := s.MinWALID(), int64(10); got != want { + if got, want := s.MinWALID, int64(10); got != want { t.Fatalf("Base()=%d, want %d", got, want) - } else if got, want := s.PageN(), 0; got != want { + } else if got, want := s.PageN, 0; got != want { t.Fatalf("PageN()=%d, want %d", got, want) } }) @@ -40,6 +40,7 @@ func TestWALSegment_Open(t *testing.T) { // TODO(BBJ): Test open w/ partially written pages. } +/* func TestWALSegment_WritePage(t *testing.T) { rand := rand.New(rand.NewSource(0)) s := MustOpenWALSegment(t, 10) @@ -84,6 +85,7 @@ func TestWALSegment_WritePage(t *testing.T) { t.Fatal("unexpected second page") } } +*/ func TestFormatWALSegmentPath(t *testing.T) { if got, want := rbf.FormatWALSegmentPath(1234), "00000000000004d2.wal"; got != want { @@ -107,6 +109,7 @@ func TestParseWALSegmentPath(t *testing.T) { }) } +/* func BenchmarkWALSegment_WriteWALPage(b *testing.B) { b.Run("8KB", func(b *testing.B) { benchmarkWALSegment_WriteWALPage(b, 8*(1<<10)) }) b.Run("16KB", func(b *testing.B) { benchmarkWALSegment_WriteWALPage(b, 16*(1<<10)) }) @@ -147,9 +150,10 @@ func benchmarkWALSegment_WriteWALPage(b *testing.B, flushSize int) { b.SetBytes(rbf.MaxWALSegmentFileSize) } +*/ // MustOpenWALSegment opens a WAL segment in a temporary path. Fails on error. -func MustOpenWALSegment(tb testing.TB, walID int64) *rbf.WALSegment { +func MustOpenWALSegment(tb testing.TB, walID int64) rbf.WALSegment { tb.Helper() dir, err := ioutil.TempDir("", "") @@ -169,11 +173,11 @@ func MustOpenWALSegment(tb testing.TB, walID int64) *rbf.WALSegment { } // MustCloseWALSegment closes s. Fails on error. -func MustCloseWALSegment(tb testing.TB, s *rbf.WALSegment) { +func MustCloseWALSegment(tb testing.TB, s rbf.WALSegment) { tb.Helper() if err := s.Close(); err != nil { tb.Fatal(err) - } else if err := os.Remove(s.Path()); err != nil { + } else if err := os.Remove(s.Path); err != nil { tb.Fatal(err) } } diff --git a/server.go b/server.go index a2ceadc01..be0a41aa3 100644 --- a/server.go +++ b/server.go @@ -269,6 +269,14 @@ func OptServerClusterDisabled(disabled bool, hosts []string) ServerOption { } } +// OptServerClusterName sets the human-readable cluster name. +func OptServerClusterName(name string) ServerOption { + return func(s *Server) error { + s.cluster.Name = name + return nil + } +} + // OptServerSerializer is a functional option on Server // used to set the serializer. func OptServerSerializer(ser Serializer) ServerOption { @@ -534,10 +542,12 @@ func (s *Server) Close() error { s.wg.Wait() var errh error + var errhs error var errc error if s.cluster != nil { errc = s.cluster.close() } + errhs = s.syncer.stopTranslationSync() if s.holder != nil { errh = s.holder.Close() } @@ -553,6 +563,9 @@ func (s *Server) Close() error { if errh != nil { return errors.Wrap(errh, "closing holder") } + if errhs != nil { + return errors.Wrap(errhs, "terminating holder translation sync") + } if errc != nil { return errors.Wrap(errc, "closing cluster") } @@ -566,7 +579,7 @@ func (s *Server) loadNodeID() string { if s.nodeID != "" { return s.nodeID } - nodeID, err := s.holder.loadNodeID() + nodeID, err := s.holder.LoadNodeID() if err != nil { s.logger.Printf("loading NodeID: %v", err) return s.nodeID diff --git a/server/cluster_test.go b/server/cluster_test.go index f1a73a921..1cbda5ffc 100644 --- a/server/cluster_test.go +++ b/server/cluster_test.go @@ -25,9 +25,8 @@ import ( "testing" "time" - "github.com/pilosa/pilosa/v2/server" - "github.com/pilosa/pilosa/v2" + "github.com/pilosa/pilosa/v2/server" "github.com/pilosa/pilosa/v2/test" "golang.org/x/sync/errgroup" ) diff --git a/server/config.go b/server/config.go index c51a7f95f..3e8575c81 100644 --- a/server/config.go +++ b/server/config.go @@ -122,6 +122,7 @@ type Config struct { Coordinator bool `toml:"coordinator"` ReplicaN int `toml:"replicas"` Hosts []string `toml:"hosts"` + Name string `toml:"name"` // TODO(2.0) move this out of cluster. (why is it here??) LongQueryTime toml.Duration `toml:"long-query-time"` } `toml:"cluster"` diff --git a/server/grpc.go b/server/grpc.go index ec99f0685..b25b48e84 100644 --- a/server/grpc.go +++ b/server/grpc.go @@ -26,6 +26,7 @@ import ( "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/logger" pb "github.com/pilosa/pilosa/v2/proto" + vdsm_pb "github.com/pilosa/pilosa/v2/proto/vdsm" "github.com/pilosa/pilosa/v2/stats" "github.com/pkg/errors" "google.golang.org/grpc" @@ -125,48 +126,6 @@ func errToStatusError(err error) error { return status.Error(codes.Unknown, err.Error()) } -// GetVDSs returns a single VDS given a name -func (h *GRPCHandler) GetVDS(ctx context.Context, req *pb.GetVDSRequest) (*pb.GetVDSResponse, error) { - // TODO: Return all schema information associated with the VDS. - // It's obviously not very useful to return the same data as given. - schema := h.api.Schema(ctx) - for _, index := range schema { - if req.Name == index.Name { - return &pb.GetVDSResponse{Vds: &pb.VDS{Name: index.Name}}, nil - } - } - return nil, status.Error(codes.NotFound, fmt.Sprintf("VDS with name %s not found", req.Name)) -} - -// GetVDSs returns a list of all VDSs -func (h *GRPCHandler) GetVDSs(ctx context.Context, req *pb.GetVDSsRequest) (*pb.GetVDSsResponse, error) { - schema := h.api.Schema(ctx) - vdss := make([]*pb.VDS, len(schema)) - for i, index := range schema { - vdss[i] = &pb.VDS{Name: index.Name} - } - return &pb.GetVDSsResponse{Vdss: vdss}, nil -} - -// PostVDS creates a new VDS -func (h *GRPCHandler) PostVDS(ctx context.Context, req *pb.PostVDSRequest) (*pb.PostVDSResponse, error) { - opts := pilosa.IndexOptions{Keys: req.Keys, TrackExistence: req.TrackExistence} - _, err := h.api.CreateIndex(ctx, req.Name, opts) - if err != nil { - return nil, errToStatusError(err) - } - return &pb.PostVDSResponse{}, nil -} - -// DeleteVDS deletes a VDS -func (h *GRPCHandler) DeleteVDS(ctx context.Context, req *pb.DeleteVDSRequest) (*pb.DeleteVDSResponse, error) { - err := h.api.DeleteIndex(ctx, req.Name) - if err != nil { - return nil, errToStatusError(err) - } - return &pb.DeleteVDSResponse{}, nil -} - func (h *GRPCHandler) execSQL(ctx context.Context, queryStr string) (pb.ToRowser, error) { h.stats.Count(pilosa.MetricSqlQueries, 1, 1) return execSQL(ctx, h.api, h.logger, queryStr) @@ -292,6 +251,103 @@ func (h *GRPCHandler) QueryPQLUnary(ctx context.Context, req *pb.QueryPQLRequest return table, errToStatusError(nil) } +// VDSMGRPCHandler contains methods which handle the various gRPC requests, ported from VDSM. +type VDSMGRPCHandler struct { + grpcHandler *GRPCHandler + api *pilosa.API + logger logger.Logger + stats stats.StatsClient +} + +func NewVDSMGRPCHandler(grpcHandler *GRPCHandler, api *pilosa.API) *VDSMGRPCHandler { + return &VDSMGRPCHandler{grpcHandler: grpcHandler, api: api, logger: logger.NopLogger, stats: stats.NopStatsClient} +} + +func (h *VDSMGRPCHandler) WithLogger(logger logger.Logger) *VDSMGRPCHandler { + h.logger = logger + return h +} + +func (h *VDSMGRPCHandler) WithStats(stats stats.StatsClient) *VDSMGRPCHandler { + h.stats = stats + return h +} + +// GetVDSs returns a single VDS given a name +func (h *VDSMGRPCHandler) GetVDS(ctx context.Context, req *vdsm_pb.GetVDSRequest) (*vdsm_pb.GetVDSResponse, error) { + typedIdOrName := req.GetIdOrName() + switch idOrName := typedIdOrName.(type) { + case *vdsm_pb.GetVDSRequest_Id: + return nil, status.Error(codes.InvalidArgument, "VDS IDs are no longer supported") + case *vdsm_pb.GetVDSRequest_Name: + schema := h.api.Schema(ctx) + for _, index := range schema { + if idOrName.Name == index.Name { + return &vdsm_pb.GetVDSResponse{Vds: &vdsm_pb.VDS{Name: index.Name}}, nil + } + } + return nil, status.Error(codes.NotFound, fmt.Sprintf("VDS with name %s not found", idOrName.Name)) + default: + return nil, status.Error(codes.NotFound, "VDS not found") + } +} + +// GetVDSs returns a list of all VDSs +func (h *VDSMGRPCHandler) GetVDSs(ctx context.Context, req *vdsm_pb.GetVDSsRequest) (*vdsm_pb.GetVDSsResponse, error) { + schema := h.api.Schema(ctx) + vdss := make([]*vdsm_pb.VDS, len(schema)) + for i, index := range schema { + vdss[i] = &vdsm_pb.VDS{Name: index.Name} + } + return &vdsm_pb.GetVDSsResponse{Vdss: vdss}, nil +} + +// PostVDS creates a new VDS +func (*VDSMGRPCHandler) PostVDS(ctx context.Context, req *vdsm_pb.PostVDSRequest) (*vdsm_pb.PostVDSResponse, error) { + // Pilosa doesn't implement VDSD files, so this is unimplemented + return nil, status.Errorf(codes.Unimplemented, "method PostVDS not implemented") +} + +// DeleteVDS deletes a VDS +func (h *VDSMGRPCHandler) DeleteVDS(ctx context.Context, req *vdsm_pb.DeleteVDSRequest) (*vdsm_pb.DeleteVDSResponse, error) { + typedIdOrName := req.GetIdOrName() + switch idOrName := typedIdOrName.(type) { + case *vdsm_pb.DeleteVDSRequest_Id: + return nil, status.Error(codes.InvalidArgument, "VDS IDs are no longer supported") + case *vdsm_pb.DeleteVDSRequest_Name: + err := h.api.DeleteIndex(ctx, idOrName.Name) + if err != nil { + return nil, errToStatusError(err) + } + return &vdsm_pb.DeleteVDSResponse{}, nil + default: + return nil, status.Error(codes.NotFound, "") + } +} + +func (h *VDSMGRPCHandler) QuerySQL(req *pb.QuerySQLRequest, srv vdsm_pb.Molecula_QuerySQLServer) error { + return h.grpcHandler.QuerySQL(req, srv) +} + +func (h *VDSMGRPCHandler) QuerySQLUnary(ctx context.Context, req *pb.QuerySQLRequest) (*pb.TableResponse, error) { + return h.grpcHandler.QuerySQLUnary(ctx, req) +} + +func (h *VDSMGRPCHandler) QueryPQL(req *vdsm_pb.QueryPQLRequest, srv vdsm_pb.Molecula_QueryPQLServer) error { + preq := &pb.QueryPQLRequest{Index: req.Vds, Pql: req.Pql} + return h.grpcHandler.QueryPQL(preq, srv) +} + +func (h *VDSMGRPCHandler) QueryPQLUnary(ctx context.Context, req *vdsm_pb.QueryPQLRequest) (*pb.TableResponse, error) { + preq := &pb.QueryPQLRequest{Index: req.Vds, Pql: req.Pql} + return h.grpcHandler.QueryPQLUnary(ctx, preq) +} + +func (h *VDSMGRPCHandler) Inspect(req *vdsm_pb.InspectRequest, srv vdsm_pb.Molecula_InspectServer) error { + preq := &pb.InspectRequest{Index: req.Vds, Columns: req.Records, FilterFields: req.FilterFields, Limit: req.Limit, Offset: req.Offset, Query: req.Query} + return h.grpcHandler.Inspect(preq, srv) +} + // ResultUint64 is a wrapper around a uint64 result type // so that we can implement the ToTabler and ToRowser // interfaces. @@ -829,6 +885,8 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe return nil } cols = limitedCols + } else { + return errors.Errorf("expected 1 result for inspect query; got %d from %q", len(resp.Results), req.Query) } } @@ -870,7 +928,13 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: &pb.ColumnResponse_Uint64ArrayVal{Uint64ArrayVal: &pb.Uint64Array{Vals: ids.Rows}}}) colAdded++ + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } case "mutex": @@ -902,6 +966,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } case "int": @@ -938,7 +1005,13 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe value = vals[0] exists = true } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } } else { value, exists, err = field.StringValue(tx, id) @@ -975,6 +1048,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } } @@ -999,6 +1075,9 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } case "bool": @@ -1030,9 +1109,12 @@ func (h *GRPCHandler) Inspect(req *pb.InspectRequest, stream pb.Pilosa_InspectSe rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } + } else { + rowResp.Columns = append(rowResp.Columns, + &pb.ColumnResponse{ColumnVal: nil}) } - case "time": + default: rowResp.Columns = append(rowResp.Columns, &pb.ColumnResponse{ColumnVal: nil}) } @@ -1144,7 +1226,9 @@ func (s *grpcServer) Serve(tlsConfig *tls.Config) error { // create grpc server s.mu.Lock() s.grpcServer = grpc.NewServer(opts...) - pb.RegisterPilosaServer(s.grpcServer, NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats)) + grpcHandler := NewGRPCHandler(s.api).WithLogger(s.logger).WithStats(s.stats) + pb.RegisterPilosaServer(s.grpcServer, grpcHandler) + vdsm_pb.RegisterMoleculaServer(s.grpcServer, NewVDSMGRPCHandler(grpcHandler, s.api).WithLogger(s.logger).WithStats(s.stats)) // register the server so its services are available to grpc_cli and others reflection.Register(s.grpcServer) diff --git a/server/handler_test.go b/server/handler_test.go index 4a1323452..47e4698b0 100644 --- a/server/handler_test.go +++ b/server/handler_test.go @@ -381,6 +381,20 @@ func TestHandler_Endpoints(t *testing.T) { } }) + t.Run("UI/usage", func(t *testing.T) { + w := httptest.NewRecorder() + h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/ui/usage", nil)) + if w.Code != gohttp.StatusOK { + t.Fatalf("unexpected status code: %d", w.Code) + } + ret := mustJSONDecode(t, w.Body) + usage := ret["bytesOnDisk"].(map[string]interface{}) + indexes := usage["indexes"].(map[string]interface{}) + if len(indexes) != 2 { + t.Fatalf("wrong length index size list: %#v", indexes) + } + }) + t.Run("Metrics", func(t *testing.T) { w := httptest.NewRecorder() h.ServeHTTP(w, test.MustNewHTTPRequest("GET", "/metrics", nil)) diff --git a/server/server.go b/server/server.go index 1b7dea316..180fd4334 100644 --- a/server/server.go +++ b/server/server.go @@ -37,6 +37,7 @@ import ( "golang.org/x/sync/errgroup" + "github.com/pelletier/go-toml" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/boltdb" "github.com/pilosa/pilosa/v2/encoding/proto" @@ -406,6 +407,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerGRPCURI(advertiseGRPCURI), pilosa.OptServerInternalClient(http.NewInternalClientFromURI(uri, c)), pilosa.OptServerClusterDisabled(m.Config.Cluster.Disabled, m.Config.Cluster.Hosts), + pilosa.OptServerClusterName(m.Config.Cluster.Name), pilosa.OptServerSerializer(proto.Serializer{}), pilosa.OptServerTxsrc(m.Config.Txsrc), coordinatorOpt, @@ -621,3 +623,10 @@ func (f *filteredWriter) Write(p []byte) (n int, err error) { } return len(p), nil } + +// ParseConfig parses s into a Config. +func ParseConfig(s string) (Config, error) { + var c Config + err := toml.Unmarshal([]byte(s), &c) + return c, err +} diff --git a/server/server_test.go b/server/server_test.go index ca1ea0e3e..270fdcaf5 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -31,7 +31,6 @@ import ( "testing" "time" - "github.com/pelletier/go-toml" "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/http" "github.com/pilosa/pilosa/v2/pql" @@ -338,7 +337,7 @@ func TestMain_MinMaxFloat(t *testing.T) { // Ensure the host can be parsed. func TestConfig_Parse_Host(t *testing.T) { - if c, err := ParseConfig(`bind = "local"`); err != nil { + if c, err := server.ParseConfig(`bind = "local"`); err != nil { t.Fatal(err) } else if c.Bind != "local" { t.Fatalf("unexpected host: %s", c.Bind) @@ -347,7 +346,7 @@ func TestConfig_Parse_Host(t *testing.T) { // Ensure the data directory can be parsed. func TestConfig_Parse_DataDir(t *testing.T) { - if c, err := ParseConfig(`data-dir = "/tmp/foo"`); err != nil { + if c, err := server.ParseConfig(`data-dir = "/tmp/foo"`); err != nil { t.Fatal(err) } else if c.DataDir != "/tmp/foo" { t.Fatalf("unexpected data dir: %s", c.DataDir) @@ -600,13 +599,6 @@ func GenerateSetCommands(n int, rand *rand.Rand) []SetCommand { return cmds } -// ParseConfig parses s into a Config. -func ParseConfig(s string) (server.Config, error) { - var c server.Config - err := toml.Unmarshal([]byte(s), &c) - return c, err -} - // MustMarshalJSON marshals v into a string. Panic on error. func MustMarshalJSON(v interface{}) string { buf, err := json.Marshal(v) diff --git a/test/cluster.go b/test/cluster.go index 41b1cea13..c71e3e68f 100644 --- a/test/cluster.go +++ b/test/cluster.go @@ -248,7 +248,7 @@ func newCluster(tb testing.TB, size int, opts ...[]server.CommandOption) (*Clust commandOpts = opts[i%len(opts)] } m := NewCommandNode(tb, i == 0, commandOpts...) - err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name+"_"+strconv.Itoa(i)), 0600) + err := ioutil.WriteFile(path.Join(m.Config.DataDir, ".id"), []byte(name+"__"+strconv.Itoa(i)), 0600) if err != nil { return nil, errors.Wrap(err, "writing node id") } diff --git a/test/pilosa.go b/test/pilosa.go index be0aaa1fc..d7fdf7aaa 100644 --- a/test/pilosa.go +++ b/test/pilosa.go @@ -302,7 +302,7 @@ func Do(t *testing.T, method, urlStr string, body string) *httpResponse { // set a timeout instead of allowing gohttp.Defaultclient to // potentially hang forever. hc := &gohttp.Client{ - Timeout: time.Second * 10, + Timeout: time.Second * 30, } resp, err := hc.Do(req) diff --git a/tournament.sh b/tournament.sh index 2b2a559f8..31c3d42c3 100755 --- a/tournament.sh +++ b/tournament.sh @@ -8,5 +8,6 @@ for i in rbf lmdb roaring rbf_lmdb rbf_roaring lmdb_rbf lmdb_roaring roaring_rbf roaring_lmdb ; do echo "$(date) starting ${i}, output to tourna.log.${i}" echo "***=== ${i} ====================*** $(date)" &> tourna.log.${i} - PILOSA_TXSRC=${i} make testv-race &>> tourna.log.${i} + PILOSA_TXSRC=${i} make testv-race 2>&1 > tourna.log.${i} done + diff --git a/translate.go b/translate.go index 101a854a3..0f8169e5c 100644 --- a/translate.go +++ b/translate.go @@ -17,6 +17,7 @@ package pilosa import ( "context" "encoding/json" + "fmt" "io" "io/ioutil" "sync" @@ -90,7 +91,15 @@ type TranslateStore interface { // the read payload. ReadFrom(io.Reader) (int64, error) - ComputeTranslatorSummary() (sum *TranslatorSummary, err error) + ComputeTranslatorSummaryRows() (sum *TranslatorSummary, err error) + ComputeTranslatorSummaryCols(partitionID int, topo *Topology) (sum *TranslatorSummary, err error) + + KeyWalker(walk func(key string, col uint64)) error + IDWalker(walk func(key string, col uint64)) error + + RepairKeys(topo *Topology, verbose, applyKeyRepairs bool) (changed bool, err error) + + GetStorePath() string } // TranslatorSummary is returned, for example from the boltdb string key translators, @@ -101,6 +110,14 @@ type TranslatorSummary struct { // ParitionID is filled for column keys PartitionID int + NodeID string + StorePath string + IsPrimary bool + IsReplica bool + + // PrimaryNodeIndex indexes into the cluster []node array to find the primary + PrimaryNodeIndex int + // Field is filled for row keys Field string @@ -112,6 +129,41 @@ type TranslatorSummary struct { // IDCount has the number of ID->Key mappings IDCount int + + // false for RowIDs, true for string-Key column IDs. + IsColKey bool +} + +func (s *TranslatorSummary) String() string { + return fmt.Sprintf(` +TranslatorSummary{ + Index : %v + PartitionID: %v + NodeID : %v + StorePath : %v + IsPrimary : %v + IsReplica : %v + PrimaryNodeIndex: %v + Field : %v + Checksum: %v + KeyCount: %v + IDCount : %v + IsColKey: %v +} +`, + s.Index, + s.PartitionID, + s.NodeID, + s.StorePath, + s.IsPrimary, + s.IsReplica, + s.PrimaryNodeIndex, + s.Field, + s.Checksum, + s.KeyCount, + s.IDCount, + s.IsColKey, + ) } // OpenTranslateStoreFunc represents a function for instantiating and opening a TranslateStore. @@ -127,7 +179,7 @@ func GenerateNextPartitionedID(index string, prev uint64, partitionID, partition // Try to use the next ID if it is in the same partition. // Otherwise find ID in next shard that has a matching partition. for id := prev + 1; ; id += ShardWidth { - if shardPartition(index, id/ShardWidth, partitionN) == partitionID { + if shardToShardPartition(index, id/ShardWidth, partitionN) == partitionID { return id } } @@ -304,6 +356,30 @@ func NewInMemTranslateStore(index, field string, partitionID, partitionN int) *I } } +func (s *InMemTranslateStore) GetStorePath() string { + return "" +} + +// KeyWalker executes walk for every pair in the database +func (s *InMemTranslateStore) KeyWalker(walk func(key string, col uint64)) error { + s.mu.RLock() + defer s.mu.RUnlock() + for id, key := range s.keysByID { + walk(key, id) + } + return nil +} + +// IDWalker executes walk for every pair in the database +func (s *InMemTranslateStore) IDWalker(walk func(key string, col uint64)) error { + s.mu.RLock() + defer s.mu.RUnlock() + for key, id := range s.idsByKey { + walk(key, id) + } + return nil +} + var _ OpenTranslateStoreFunc = OpenInMemTranslateStore // OpenInMemTranslateStore returns a new instance of InMemTranslateStore. @@ -312,9 +388,18 @@ func OpenInMemTranslateStore(rawurl, index, field string, partitionID, partition return NewInMemTranslateStore(index, field, partitionID, partitionN), nil } -func (s *InMemTranslateStore) ComputeTranslatorSummary() (sum *TranslatorSummary, err error) { +func (s *InMemTranslateStore) ComputeTranslatorSummaryRows() (sum *TranslatorSummary, err error) { panic("TODO") } + +func (s *InMemTranslateStore) ComputeTranslatorSummaryCols(partitionID int, topo *Topology) (sum *TranslatorSummary, err error) { + panic("TODO") +} + +func (s *InMemTranslateStore) RepairKeys(topo *Topology, verbose, applyKeyRepairs bool) (changed bool, err error) { + panic("TODO") +} + func (s *InMemTranslateStore) Close() error { return nil } diff --git a/txfactory_internal_test.go b/txfactory_internal_test.go index dd8225aa7..d172099d5 100644 --- a/txfactory_internal_test.go +++ b/txfactory_internal_test.go @@ -95,9 +95,9 @@ func Test_TxFactory_Qcx_query_context(t *testing.T) { // allow all goro to finish before Closing the lmdb.env, otherwise // we will crash as the goroutines making Tx will try to use the env // after it is closed. It can take quite a while. - // one writer might be blocking the other... so ask for only N-1 at first. - barrier.BlockUntil(N - 1) - //barrier.BlockUntil(N) + // one writer might be blocking the other... so ask for only N-2 at first + // to avoid deadlock. + barrier.BlockUntil(N - 2) barrier.UnblockReaders() time.Sleep(1 * time.Second) } diff --git a/utils_internal_test.go b/utils_internal_test.go index 91f2c0745..195c53cef 100644 --- a/utils_internal_test.go +++ b/utils_internal_test.go @@ -70,7 +70,7 @@ func NewTestCluster(tb testing.TB, n int) *cluster { c.ReplicaN = 1 c.Hasher = NewTestModHasher() c.Path = path - c.Topology = newTopology() + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) for i := 0; i < n; i++ { c.nodes = append(c.nodes, &Node{ @@ -137,6 +137,15 @@ func (t *ClusterCluster) CreateIndex(name string) error { return nil } +func (t *ClusterCluster) CreateIndexWithOpt(name string, opt IndexOptions) error { + for _, c := range t.Clusters { + if _, err := c.holder.CreateIndexIfNotExists(name, opt); err != nil { + return err + } + } + return nil +} + func (t *ClusterCluster) CreateField(index, field string, opts FieldOption) error { for _, c := range t.Clusters { idx, err := c.holder.CreateIndexIfNotExists(index, IndexOptions{}) @@ -272,7 +281,8 @@ func (t *ClusterCluster) addCluster(i int, saveTopology bool) (*cluster, error) c.ReplicaN = 1 c.Hasher = NewTestModHasher() c.Path = path - c.Topology = newTopology() + c.partitionN = DefaultPartitionN + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) c.holder = h c.Node = node c.Coordinator = t.common.Nodes[0].ID // the first node is the coordinator @@ -515,3 +525,47 @@ func (t *ClusterCluster) FollowResizeInstruction(instr *ResizeInstruction) error node := instr.Coordinator return bcast{t: t}.SendTo(node, complete) } + +var _ = NewTestClusterWithReplication // happy linter + +func NewTestClusterWithReplication(tb testing.TB, nNodes, nReplicas, partitionN int) (c *cluster, cleaner func()) { + path, err := testhook.TempDir(tb, "pilosa-cluster-") + if err != nil { + panic(err) + } + + // holder + h := NewHolder(path, nil) + + // cluster + availableShardFileFlushDuration.Set(100 * time.Millisecond) + c = newCluster() + c.holder = h + c.ReplicaN = nReplicas + c.Hasher = &Jmphasher{} + c.Path = path + c.partitionN = partitionN + c.Topology = NewTopology(c.Hasher, c.partitionN, c.ReplicaN, c) + + for i := 0; i < nNodes; i++ { + nodeID := fmt.Sprintf("node%d", i) + c.nodes = append(c.nodes, &Node{ + ID: nodeID, + URI: NewTestURI("http", fmt.Sprintf("host%d", i), uint16(0)), + }) + c.Topology.addID(nodeID) + } + + c.Node = c.nodes[0] + c.Coordinator = c.nodes[0].ID + c.SetState(ClusterStateNormal) + + if err := c.holder.Open(); err != nil { + panic(err) + } + + return c, func() { + c.holder.Close() + c.close() + } +}