diff --git a/bolt.go b/bolt.go index 9db81fdca..39106fecf 100644 --- a/bolt.go +++ b/bolt.go @@ -441,7 +441,7 @@ func (tx *BoltTx) Type() string { } func (tx *BoltTx) UseRowCache() bool { - return rbf.EnableRowCache + return rbf.EnableRowCache() } // Pointer gives us a memory address for the underlying transaction for debugging. diff --git a/ctl/server.go b/ctl/server.go index 221f3918f..204a368bf 100644 --- a/ctl/server.go +++ b/ctl/server.go @@ -17,6 +17,7 @@ package ctl import ( "time" + "github.com/pilosa/pilosa/v2" "github.com/pilosa/pilosa/v2/server" "github.com/spf13/cobra" ) @@ -88,7 +89,10 @@ func BuildServerFlags(cmd *cobra.Command, srv *server.Command) { flags.IntVar(&srv.Config.Profile.MutexFraction, "profile.mutex-fraction", srv.Config.Profile.MutexFraction, "Sampling fraction for mutex contention profiling. Sample 1/ of events.") // Transactional storage engine - flags.StringVarP(&srv.Config.Txsrc, "tx", "", "", "transaction/storage to use: one of roaring, rbf, badger, rbf_roaring, roaring_rbf, badger_roaring, roaring_badger, badger_rbf, or rbf_badger (default roaring)") + flags.StringVarP(&srv.Config.Txsrc, "tx", "", pilosa.DefaultTxsrc, "transaction/storage to use: one of roaring, rbf, bolt, lmdb, or a blue-green setup: rbf_roaring, roaring_rbf, bolt_roaring, roaring_bolt, bolt_rbf, etc.") + + // RowcacheOff + flags.BoolVarP((&srv.Config.RowcacheOff), "rowcache-off", "", srv.Config.RowcacheOff, "turn off the rowcache for all backends (reduces memory use)") // Postgres endpoint flags.StringVar(&srv.Config.Postgres.Bind, "postgres.bind", srv.Config.Postgres.Bind, "Address to which to bind a postgres endpoint (leave blank to disable)") diff --git a/go.mod b/go.mod index 54169033d..bfd870b97 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/hashicorp/memberlist v0.1.3 github.com/improbable-eng/grpc-web v0.13.0 github.com/lib/pq v1.8.0 + github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b github.com/opentracing/opentracing-go v1.1.0 github.com/pelletier/go-toml v1.2.0 github.com/pkg/errors v0.9.1 diff --git a/go.sum b/go.sum index f160e1afe..ea9c803a4 100644 --- a/go.sum +++ b/go.sum @@ -148,6 +148,8 @@ github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQz github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b h1:cZADDaNYM7xn/nklO3g198JerGQjadFuA0ofxBJgK0Y= +github.com/molecula/apophenia v0.0.0-20190827192002-68b7a14a478b/go.mod h1:uXd1BiH7xLmgkhVmspdJLENv6uGWrTL/MQX2TN7Yz9s= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= diff --git a/holder.go b/holder.go index bece8baf2..fdaabf099 100644 --- a/holder.go +++ b/holder.go @@ -30,6 +30,7 @@ import ( "time" "github.com/pilosa/pilosa/v2/logger" + "github.com/pilosa/pilosa/v2/rbf" "github.com/pilosa/pilosa/v2/roaring" "github.com/pilosa/pilosa/v2/stats" "github.com/pilosa/pilosa/v2/testhook" @@ -137,6 +138,9 @@ type HolderOpts struct { // Txsrc controls the tx/storage engine we instatiate. Set by // server.go OptServerTxsrc Txsrc string + + // RowcacheOff, if true, turns off the row cache for all storage backends. + RowcacheOff bool } func (h *Holder) StartTransaction(ctx context.Context, id string, timeout time.Duration, exclusive bool) (*Transaction, error) { @@ -196,6 +200,7 @@ type HolderConfig struct { NewAttrStore func(string) AttrStore Logger logger.Logger Txsrc string + RowcacheOff bool } func DefaultHolderConfig() *HolderConfig { @@ -243,7 +248,7 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { OpenTransactionStore: cfg.OpenTransactionStore, translationSyncer: cfg.TranslationSyncer, Logger: cfg.Logger, - Opts: HolderOpts{Txsrc: cfg.Txsrc}, + Opts: HolderOpts{Txsrc: cfg.Txsrc, RowcacheOff: cfg.RowcacheOff}, SnapshotQueue: defaultSnapshotQueue, @@ -254,6 +259,8 @@ func NewHolder(path string, cfg *HolderConfig) *Holder { indexes: make(map[string]*Index), } + rbf.SetRowcacheOn(!cfg.RowcacheOff) + txf, err := NewTxFactory(cfg.Txsrc, path, h) panicOn(err) h.txf = txf diff --git a/license.exceptions b/license.exceptions index 46230d311..450a591d2 100644 --- a/license.exceptions +++ b/license.exceptions @@ -6,6 +6,7 @@ ./lru/lru.go ./roaring/btree.go ./roaring/btree_test.go +./roaring/containerarchetype_string.go ./proto/pilosa.pb.go ./logger/filewriter.go ./logger/filewriter_test.go diff --git a/lmdb.go b/lmdb.go index dbe539897..e952df65d 100644 --- a/lmdb.go +++ b/lmdb.go @@ -551,7 +551,7 @@ func (tx *LMDBTx) Type() string { } func (tx *LMDBTx) UseRowCache() bool { - return rbf.EnableRowCache + return rbf.EnableRowCache() } // Pointer gives us a memory address for the underlying transaction for debugging. diff --git a/rbf.go b/rbf.go index 0e5b1ec2a..deb427b8e 100644 --- a/rbf.go +++ b/rbf.go @@ -449,7 +449,7 @@ func (tx *RBFTx) UseRowCache() bool { // the rowCache without first making a copy. // So we only use the rowCache if the copy is // enabled. - return rbf.EnableRowCache + return rbf.EnableRowCache() } // rbfName returns a NULL-separated key used for identifying bitmap maps in RBF. diff --git a/rbf/cursorx.go b/rbf/cursorx.go index e196f5c7f..3c4b16352 100644 --- a/rbf/cursorx.go +++ b/rbf/cursorx.go @@ -19,6 +19,7 @@ import ( "io" "math" "os" + "sync/atomic" "github.com/pilosa/pilosa/v2/roaring" "github.com/pkg/errors" @@ -26,7 +27,21 @@ import ( // if enableRowCache, then we must not return mmap-ed memory // directly, but only a copy. -const EnableRowCache = true +var enableRowcache int64 = 1 + +// SetEnableRowCache should only be called in NewHolder before +// all other reads. +func SetRowcacheOn(on bool) { + if on { + atomic.StoreInt64(&enableRowcache, 1) + } else { + atomic.StoreInt64(&enableRowcache, 0) + } +} + +func EnableRowCache() bool { + return atomic.LoadInt64(&enableRowcache) == 1 +} //probably should just implement the container interface // but for now i'll do it @@ -151,7 +166,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) { orig := l.Data var cpMaybe []byte var mapped bool - if EnableRowCache || tx.db.DoAllocZero { + if EnableRowCache() || tx.db.DoAllocZero { // make a copy, otherwise the rowCache will see corrupted data // or mmapped data that may disappear. cpMaybe = make([]byte, len(orig)) @@ -168,7 +183,7 @@ func toContainer(l leafCell, tx *Tx) (c *roaring.Container) { case ContainerTypeBitmapPtr: _, bm, _ := tx.leafCellBitmap(toPgno(cpMaybe)) cloneMaybe := bm - if EnableRowCache { + if EnableRowCache() { cloneMaybe = make([]uint64, len(bm)) copy(cloneMaybe, bm) } diff --git a/roaring/benchpretty/main.go b/roaring/benchpretty/main.go new file mode 100644 index 000000000..424f9c948 --- /dev/null +++ b/roaring/benchpretty/main.go @@ -0,0 +1,128 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bufio" + "fmt" + "log" + "os" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/pilosa/pilosa/v2/roaring" +) + +var pattern = regexp.MustCompile(`^BenchmarkCtOps/([^/]+)/([^/]+)/([^-]+)-([0-9]+)\s*([0-9]+)\s*([0-9.]+) ns/op`) + +func parseFile(path string, benchmarks map[string]map[string]map[string]float64, known map[string]bool) error { + // unset all the seen flags in the known map. if we end up with any unseen, the + // benchmarks are incomplete. + for k := range known { + known[k] = false + } + file, err := os.Open(path) + if err != nil { + return err + } + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if !strings.HasPrefix(line, "BenchmarkCtOps/") { + continue + } + matches := pattern.FindStringSubmatch(line) + if matches == nil { + return fmt.Errorf("can't parse line: '%s'", line) + } + t1, t2 := matches[1], matches[2] + if _, ok := known[t1]; !ok { + return fmt.Errorf("unknown archetype '%s'", t1) + } + if _, ok := known[t2]; !ok { + return fmt.Errorf("unknown archetype '%s'", t2) + } + known[t1] = true + known[t2] = true + op := matches[3] + time, err := strconv.ParseFloat(matches[6], 64) + if err != nil { + return fmt.Errorf("parsing float [%s]: %v", matches[6], err) + } + if benchmarks[op] == nil { + benchmarks[op] = make(map[string]map[string]float64, 16) + } + if benchmarks[op][t1] == nil { + benchmarks[op][t1] = make(map[string]float64, 16) + } + benchmarks[op][t1][t2] += time + } + for k, v := range known { + if !v { + fmt.Printf("warning: container archetype '%s' missing in benchmarks\n", k) + } + } + return nil +} + +func main() { + maxLen := 0 + + knownArchetypes := make(map[string]bool, len(roaring.ContainerArchetypeNames)) + for _, name := range roaring.ContainerArchetypeNames { + if len(name) > maxLen { + maxLen = len(name) + } + knownArchetypes[name] = false + } + benchmarks := make(map[string]map[string]map[string]float64, 8) + for _, file := range os.Args[1:] { + err := parseFile(file, benchmarks, knownArchetypes) + if err != nil { + log.Fatalf("parsing '%s': %v", file, err) + } + } + if len(benchmarks) < 1 { + log.Fatalf("no benchmarks parsed?") + } + ops := make([]string, 0, 8) + for k := range benchmarks { + ops = append(ops, k) + } + sort.Strings(ops) + for _, op := range ops { + fmt.Printf("%s:\n", op) + fmt.Printf("%*s ", maxLen, "") + for _, name := range roaring.ContainerArchetypeNames { + fmt.Printf(" %*s", maxLen, name) + } + fmt.Print("\n") + for _, self := range roaring.ContainerArchetypeNames { + fmt.Printf("%*s ", maxLen, self) + for _, other := range roaring.ContainerArchetypeNames { + time, ok := benchmarks[op][self][other] + if ok { + fmt.Printf(" %*.1f", maxLen, time) + } else { + fmt.Printf(" %*s", maxLen, "--") + } + } + fmt.Print("\n") + } + fmt.Print("\n") + } +} diff --git a/roaring/container_archetypes.go b/roaring/container_archetypes.go new file mode 100644 index 000000000..2da817d99 --- /dev/null +++ b/roaring/container_archetypes.go @@ -0,0 +1,205 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roaring + +import ( + "fmt" + "math/rand" + "sort" + "strconv" + "strings" + "sync" + + "github.com/molecula/apophenia" +) + +// ContainerArchetypeNames is the list of supported container archetypes +// used in some testing. This is exported for use in a benchmark-analysis +// tool. +var ContainerArchetypeNames = []string{ + "Empty", + "Ary1", + "Ary16", + "Ary256", + "Ary512", + "Ary1024", + "Ary4096", + "RunFull", + "RunSplit", + "Run16", + "Run16Small", + "Run256", + "Run256Small", + "Run1024", + "BM512", + "BM1024", + "BM4096", + "BM4097", + "BM32768", + "BM65000", +} + +var containerArchetypes [][]*Container +var containerArchetypesErr error +var initContainerArchetypes sync.Once + +func makeArchetypalContainer(rng *rand.Rand, name string) (*Container, error) { + var c *Container + switch { + case name == "Empty": + c = NewContainerArray(nil) + case strings.HasPrefix(name, "Ary"): + size, err := strconv.Atoi(name[3:]) + if err != nil { + return nil, fmt.Errorf("can't parse array size: %v", err) + } + array := make([]uint16, size) + seq := apophenia.NewSequence(rng.Int63()) + perm, err := apophenia.NewPermutation(65536, 0, seq) + if err != nil { + return nil, err + } + for i := 0; i < size; i++ { + array[i] = uint16(perm.Next()) + } + sort.Slice(array, func(a, b int) bool { return array[a] < array[b] }) + c = NewContainerArray(array) + case name == "RunFull": + c = NewContainerRun([]Interval16{{Start: 0, Last: 65535}}) + case name == "RunSplit": + runs := []Interval16{ + {Start: 0, Last: 32700 + uint16(rng.Intn(30))}, + {Start: 32768 + uint16(rng.Intn(30)), Last: 65535}, + } + c = NewContainerRun(runs) + case strings.HasPrefix(name, "Run"): + countEnd := len(name) + small := strings.HasSuffix(name, "Small") + if small { + countEnd -= 5 + } + count, err := strconv.Atoi(name[3:countEnd]) + if err != nil { + return nil, fmt.Errorf("can't parse run count in '%s': %v", name, err) + } + runs := make([]Interval16, count) + // For size intervals, we want to divvy up the total + // space around count+1 points, and populate the space between + // those points with a run smaller than that. + stride := int32(65535 / (count + 1)) + upper := stride - 10 + lower := stride / 10 + if small { + lower = 3 + upper = lower + (stride / 20) + } + variance := upper - lower + next := int32(0) + prev := int32(0) + for i := 0; i < count; i++ { + next += stride + middle := (prev + next) / 2 + runSize := rng.Int31n(variance) + lower + offset := rng.Int31n(variance) + runs[i].Start = uint16(middle + offset - (runSize / 2)) + runs[i].Last = runs[i].Start + uint16(runSize) + if runs[i].Last < runs[i].Start { + return nil, fmt.Errorf("fatal, run %d starts at %d, tries to end at %d", i, runs[i].Start, runs[i].Last) + } + prev = next + if i > 0 { + if runs[i].Start <= runs[i-1].Last { + if runs[i-1].Last > 65533 { + return nil, fmt.Errorf("fatal, run %d starts at %d, previous run ended at %d", + i, runs[i].Start, runs[i-1].Last) + } else { + runs[i].Start = runs[i-1].Last + 2 + if runs[i].Last < runs[i].Start { + runs[i].Last = runs[i].Start + } + } + } + } + } + c = NewContainerRun(runs) + case strings.HasPrefix(name, "BM"): + size, err := strconv.Atoi(name[2:]) + if err != nil { + return nil, fmt.Errorf("can't parse bitmap size: %v", err) + } + bitmap := make([]uint64, bitmapN) + n := int32(0) + flip := false + // Picking random bits sometimes overlaps, so we want to + // keep trying until we get the requested number. But that's + // really slow for N close to the maximum, so if we want more + // than half the bits set, we'll do it backwards and then + // invert the bits. + bits := int32(size) + if size > 32768 { + flip = true + bits = 65536 - bits + } + for n < bits { + pos := (rng.Uint64() & 65535) + bit := uint64(1 << (pos & 63)) + if bitmap[pos/64]&bit == 0 { + n++ + bitmap[pos/64] |= bit + } + } + if flip { + for i := 0; i < bitmapN; i++ { + bitmap[i] = ^bitmap[i] + } + n = 65536 - n + } + c = NewContainerBitmap(int(n), bitmap) + count := c.count() + if count != n { + return nil, fmt.Errorf("bitmap should have %d bits, has %d", n, count) + } + } + return c, nil +} + +// InitContainerArchetypes ensures that createContainerArchetypes has been +// called, and returns the results of that one call. +func InitContainerArchetypes() ([][]*Container, error) { + initContainerArchetypes.Do(func() { + containerArchetypes, containerArchetypesErr = createContainerArchetypes(8) + }) + return containerArchetypes, containerArchetypesErr +} + +// createContainerArchetypes creates a slice of *roaring.Container corresponding +// to each container archetype, or reports an error. +func createContainerArchetypes(count int) (cats [][]*Container, err error) { + cats = make([][]*Container, len(ContainerArchetypeNames)) + // seed is arbitrary, but picking a seed means we don't get different + // behavior for each run + rng := rand.New(rand.NewSource(23)) + for i, name := range ContainerArchetypeNames { + cats[i] = make([]*Container, count) + for j := 0; j < count; j++ { + c, err := makeArchetypalContainer(rng, name) + if err != nil { + return nil, err + } + cats[i][j] = c + } + } + return cats, nil +} diff --git a/roaring/roaring.go b/roaring/roaring.go index 8cf2587a5..15414c9ef 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -4238,21 +4238,22 @@ func intersectBitmapRun(a, b *Container) *Container { statsHit("intersect/BitmapRun") var output *Container runs := b.runs() - if b.N() <= ArrayMaxSize || a.N() <= ArrayMaxSize { - // output is array container - array := make([]uint16, 0, b.N()) + // Intersection will be array-sized for sure if either of the inputs + // is array-sized. + if b.N() <= ArrayMaxSize { + var scratch [ArrayMaxSize]uint16 + n := 0 for _, iv := range runs { - for i := iv.Start; i <= iv.Last; i++ { - if a.bitmapContains(i) { - array = append(array, i) - } - // If the run ends the container, break to avoid an infinite loop. - if i == 65535 { - break + for i := int(iv.Start); i <= int(iv.Last); i++ { + if a.bitmapContains(uint16(i)) { + scratch[n] = uint16(i) + n++ } } } - + // output is array container + array := make([]uint16, n) + copy(array, scratch[:]) output = NewContainerArray(array) } else { // right now this iterates through the runs and sets integers in the diff --git a/roaring/roaring_container_test.go b/roaring/roaring_container_test.go new file mode 100644 index 000000000..bf573e529 --- /dev/null +++ b/roaring/roaring_container_test.go @@ -0,0 +1,94 @@ +// Copyright 2019 Pilosa Corp. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package roaring + +import ( + "fmt" + "testing" +) + +type containerOp struct { + name string + fn func(a, b *Container) +} + +var containerOps = []containerOp{ + {"intersect", func(a, b *Container) { _ = intersect(a, b) }}, + {"union", func(a, b *Container) { _ = union(a, b) }}, + {"difference", func(a, b *Container) { _ = difference(a, b) }}, + {"xor", func(a, b *Container) { _ = xor(a, b) }}, + {"intersectionCount", func(a, b *Container) { _ = intersectionCount(a, b) }}, +} + +// Run each container type against each other container type. In an earlier +// implementation, this had a subtle bug; we generated two sets of archetypal +// containers, so the run of Array1 vs. Array4096 used list1's Array1, and +// list2's Array4096, and the run of Arary4096 vs Array1 used list1's Array4096 +// and list2's Array1. This created a subtle performance glitch, because +// list1 happened to have an Array1 containing 53,127 and list2 happened to +// have an Array1 containing 3,917, which meant that the second item being +// Array1 often looked dramatically faster than the first item being Array1. +// To reduce the impact of such things, we generate 8 of each container, and +// do each test on the whole 8x8 matrix. This does mean each operation is +// being run with a container compared with itself 1/8 of the time. +func BenchmarkCtOps(b *testing.B) { + ca, err := InitContainerArchetypes() + if err != nil { + b.Fatalf("creating container archetypes: %v", err) + } + for idx1, n1 := range ContainerArchetypeNames { + ca1 := ca[idx1] + for idx2, n2 := range ContainerArchetypeNames { + base := fmt.Sprintf("%s/%s", n1, n2) + ca2 := ca[idx2] + b.Run(base, func(b *testing.B) { + for _, op := range containerOps { + b.Run(op.name, func(b *testing.B) { + for i := 0; i < b.N; i++ { + for _, c1 := range ca1 { + for _, c2 := range ca2 { + op.fn(c1, c2) + } + } + } + }) + } + }) + } + } +} + +func TestIntersectVariants(t *testing.T) { + ca, err := InitContainerArchetypes() + if err != nil { + t.Fatalf("creating container archetypes: %v", err) + } + for idx1, n1 := range ContainerArchetypeNames { + ca1 := ca[idx1] + for idx2, n2 := range ContainerArchetypeNames { + ca2 := ca[idx2] + for i1, c1 := range ca1 { + for i2, c2 := range ca2 { + full := intersect(c1, c2) + count := intersectionCount(c1, c2) + if full.N() != count { + t.Errorf("intersecting %s[%d] and %s[%d]: container has N %d, count was %d", + n1, i1, n2, i2, full.N(), count) + } + } + } + } + } +} diff --git a/rrtx.go b/rrtx.go index 8e254496e..96d5c198e 100644 --- a/rrtx.go +++ b/rrtx.go @@ -63,7 +63,7 @@ func (tx *RoaringTx) Dump(short bool, shard uint64) { } func (tx *RoaringTx) UseRowCache() bool { - return rbf.EnableRowCache + return rbf.EnableRowCache() } func (tx *RoaringTx) SliceOfShards(index, field, view, optionalViewPath string) (sliceOfShards []uint64, err error) { diff --git a/server.go b/server.go index be0a41aa3..70123465a 100644 --- a/server.go +++ b/server.go @@ -343,6 +343,15 @@ func OptServerTxsrc(txsrc string) ServerOption { } } +// OptServerRowcacheOff is a functional option on Server +// used to turn off the row cache. +func OptServerRowcacheOff(rowcacheOff bool) ServerOption { + return func(s *Server) error { + s.holderConfig.RowcacheOff = rowcacheOff + return nil + } +} + // NewServer returns a new instance of Server. func NewServer(opts ...ServerOption) (*Server, error) { cluster := newCluster() @@ -397,6 +406,7 @@ func NewServer(opts ...ServerOption) (*Server, error) { } s.holder = NewHolder(path, s.holderConfig) s.holder.Stats.SetLogger(s.logger) + s.holder.Logger.Printf("RowCacheOff: %v", s.holderConfig.RowcacheOff) s.cluster.Path = path s.cluster.logger = s.logger diff --git a/server/config.go b/server/config.go index 3e8575c81..3e73669ec 100644 --- a/server/config.go +++ b/server/config.go @@ -190,8 +190,8 @@ type Config struct { // Txsrc determines which Tx implementation the holder/Index will use; one // of the available transactional-storage engines. Choices are listed // in the string constants below. Should be one of - // "roaring","badger", "rbf", "badger_roaring", "roaring_badger", "rbf_roaring", - // "roaring_rbf", "badger_rbf", "rbf_badger", or any later addition. The + // "roaring","bolt", "rbf", "bolt_roaring", "roaring_bolt", "rbf_roaring", + // "roaring_rbf", "bolt_rbf", "rbf_bolt", or any later addition. The // engines with _ underscore indicate use of a blueGreenTx with a comparison // of values back from each Tx method, and a panic if they differ. This // is an effective test for consistency. If "rbf_roaring" is specified, then @@ -199,6 +199,9 @@ type Config struct { // If "roaring_rbf" is chosen, then the RBF values are the ones actually // returned from the blueGreenTx. Txsrc string `toml:"txsrc"` + + // RowcacheOff, if true, turns off the row cache for all storage backends. + RowcacheOff bool `toml:"rowcache-off"` } // NewConfig returns an instance of Config with default options. diff --git a/server/server.go b/server/server.go index f172f5647..ce84be988 100644 --- a/server/server.go +++ b/server/server.go @@ -410,6 +410,7 @@ func (m *Command) SetupServer() error { pilosa.OptServerClusterName(m.Config.Cluster.Name), pilosa.OptServerSerializer(proto.Serializer{}), pilosa.OptServerTxsrc(m.Config.Txsrc), + pilosa.OptServerRowcacheOff(m.Config.RowcacheOff), coordinatorOpt, }