un-disable checkptr by fixing the memory problems

Step one: switch to etcd.io's bbolt fork of boltdb.

The etcd-io fork of boltdb isn't archived, and has fixes for boltdb's
interactions with checkptr, allowing us to drop the checkptr-disabling
hackery.

This seems to be a drop-in replacement; etcd/bbolt says that the file
format is "fixed" (I believe in the sense of "unchanging"), and I can
run pilosa on an existing data directory with this.

Step two:

Fix missing caps in roaring.go that were also triggering the same
issues.
This commit is contained in:
Seebs 2020-10-14 12:51:50 -05:00
parent de642ed90c
commit 76fe49d390
6 changed files with 32 additions and 32 deletions

View file

@ -14,7 +14,6 @@ LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/
GO_VERSION=latest
RELEASE ?= 0
RELEASE_ENABLED = $(subst 0,,$(RELEASE))
NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" )
BUILD_TAGS += $(if $(RELEASE_ENABLED),release)
BUILD_TAGS += shardwidth$(SHARD_WIDTH)
TEST_TAGS = roaringparanoia
@ -39,11 +38,11 @@ vendor: go.mod
# Run test suite
test:
go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v
go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v
# Run test suite with race flag
test-race:
go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v
go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v
testv: topt testvsub
@ -58,7 +57,7 @@ testvsub:
set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \
echo; echo "___ testing subpkg $$i"; \
cd $$i; pwd; \
go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \
go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \
echo; echo "999 done testing subpkg $$i"; \
cd ..; \
done
@ -67,7 +66,7 @@ testvsub-race:
set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \
echo; echo "___ testing subpkg $$i -race"; \
cd $$i; pwd; \
go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \
go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \
echo; echo "999 done testing subpkg $$i -race"; \
cd ..; \
done
@ -212,49 +211,49 @@ docker-test:
# The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt.
topt:
mv log.topt.roar log.topt.roar.prev || true
go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.roar
go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar
@echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l
@echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l
topt-badger:
mv log.topt.badger log.topt.badger.prev || true
PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger
PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.badger
@echo " log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l
@echo " log.topt.badger red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l
topt-badger-race:
mv log.topt.badger-race log.topt.badger-race.prev || true
PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger-race
PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.badger-race
@echo " log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l
@echo " log.topt.badger-race red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l
topt-rbf:
mv log.topt.rbf log.topt.rbf.prev || true
PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf
PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.rbf
@echo " log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l
@echo " log.topt.rbf red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l
topt-rbf-race:
mv log.topt.rbf-race log.topt.rbf-race.prev || true
PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -timeout 120m 2>&1 | tee log.topt.rbf-race
PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -timeout 120m 2>&1 | tee log.topt.rbf-race
@echo " log.topt.rbf-race green: \c"; cat log.topt.rbf-race | grep PASS |wc -l
@echo " log.topt.rbf-race red: \c"; cat log.topt.rbf-race | grep '\-\-\- FAIL' |wc -l
topt-lmdb:
mv log.topt.lmdb log.topt.lmdb.prev || true
PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb
PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lmdb
@echo " log.topt.lmdb green: \c"; cat log.topt.lmdb | grep PASS |wc -l
@echo " log.topt.lmdb red: \c"; cat log.topt.lmdb | grep '\-\-\- FAIL' |wc -l
topt-lmdb-race:
mv log.topt.lmdb log.topt.lmdb.prev || true
PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb-race
PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lmdb-race
@echo " log.topt.lmdb-race green: \c"; cat log.topt.lmdb-race | grep PASS |wc -l
@echo " log.topt.lmdb-race red: \c"; cat log.topt.lmdb-race | grep '\-\-\- FAIL' |wc -l
topt-race:
mv log.topt.race log.topt.race.prev || true
go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race
go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race
@echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l
@echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l
@ -263,73 +262,73 @@ topt-race:
bg-rr: # shorthand for bluegreen test with A:badger; B:roaring
mv log.bg-rr log.bg-rr.prev || true
PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rr
PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bg-rr
@echo " log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l
@echo " log.bg-rr red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l
rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned).
mv log.bg.roar_bg log.bg.roar_bg.prev || true
PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg
PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-bg
@echo " log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l
@echo " log.rr-bg red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l
rbf-rr:
mv log.rbf-rr log.rbf-rr.prev || true
PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-rr
PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-rr
@echo " log.rbf-rr green: \c"; cat log.rbf-rr | grep PASS |wc -l
@echo " log.rbf-rr red: \c"; cat log.rbf-rr | grep '\-\-\- FAIL' |wc -l
rr-rbf:
mv log.rr-rbf log.rr-rbf.prev || true
PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-rbf
PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-rbf
@echo " log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l
@echo " log.rr-rbf red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l
rbf-bg:
mv log.rbf-bg log.rbf-bg.prev || true
PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-bg
PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-bg
@echo " log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l
@echo " log.rbf-bg red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l
bg-rbf:
mv log.bg-rbf log.bg-rbf.prev || true
PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rbf
PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bg-rbf
@echo " log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l
@echo " log.bg-rbf red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l
rbf-lm:
mv log.rbf-lm log.rbf-lm.prev || true
PILOSA_TXSRC=rbf_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-lm
PILOSA_TXSRC=rbf_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-lm
@echo " log.rbf-lm green: \c"; cat log.rbf-lm | grep PASS |wc -l
@echo " log.rbf-lm red: \c"; cat log.rbf-lm | grep '\-\-\- FAIL' |wc -l
lm-rbf:
mv log.lm-rbf log.lm-rbf.prev || true
PILOSA_TXSRC=lmdb_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rbf
PILOSA_TXSRC=lmdb_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.lm-rbf
@echo " log.lm-rbf green: \c"; cat log.lm-rbf | grep PASS |wc -l
@echo " log.lm-rbf red: \c"; cat log.lm-rbf | grep '\-\-\- FAIL' |wc -l
lm-rr:
mv log.lm-rr log.lm-rr.prev || true
PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rr
PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.lm-rr
@echo " log.lm-rr green: \c"; cat log.lm-rr | grep PASS |wc -l
@echo " log.lm-rr red: \c"; cat log.lm-rr | grep '\-\-\- FAIL' |wc -l
rr-lm:
mv log.rr-lm log.rr-lm.prev || true
PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-lm
PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-lm
@echo " log.rr-lm green: \c"; cat log.rr-lm | grep PASS |wc -l
@echo " log.rr-lm red: \c"; cat log.rr-lm | grep '\-\-\- FAIL' |wc -l
bg-lm:
mv log.topt.bg-lm log.topt.bg-lm.prev || true
PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.bg-lm
PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bg-lm
@echo " log.topt.bg-lm green: \c"; cat log.topt.bg-lm | grep PASS |wc -l
@echo " log.topt.bg-lm red: \c"; cat log.topt.bg-lm | grep '\-\-\- FAIL' |wc -l
lm-bg:
mv log.topt.lm-bg log.topt.lm-bg.prev || true
PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lm-bg
PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lm-bg
@echo " log.topt.lm-bg green: \c"; cat log.topt.lm-bg | grep PASS |wc -l
@echo " log.topt.lm-bg red: \c"; cat log.topt.lm-bg | grep '\-\-\- FAIL' |wc -l

View file

@ -25,9 +25,9 @@ import (
"github.com/cespare/xxhash"
"github.com/boltdb/bolt"
"github.com/pilosa/pilosa/v2"
"github.com/pkg/errors"
bolt "go.etcd.io/bbolt"
)
// attrBlockSize is the size of attribute blocks for anti-entropy.

View file

@ -26,10 +26,10 @@ import (
"sync"
"time"
"github.com/boltdb/bolt"
"github.com/pilosa/pilosa/v2"
"github.com/pkg/errors"
"github.com/zeebo/blake3"
bolt "go.etcd.io/bbolt"
"runtime/pprof"
)

2
go.mod
View file

@ -7,7 +7,6 @@ require (
github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895
github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
github.com/benbjohnson/immutable v0.2.0
github.com/boltdb/bolt v1.3.1
github.com/cespare/xxhash v1.1.0
github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect
github.com/davecgh/go-spew v1.1.1
@ -41,6 +40,7 @@ require (
github.com/uber/jaeger-client-go v2.16.0+incompatible
github.com/uber/jaeger-lib v2.2.0+incompatible // indirect
github.com/zeebo/blake3 v0.0.4
go.etcd.io/bbolt v1.3.5
golang.org/x/mod v0.3.0
golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect
golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208

5
go.sum
View file

@ -24,8 +24,6 @@ github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylH
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0=
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4=
github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
@ -258,7 +256,10 @@ github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI=
github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34=
github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05 h1:4pW5fMvVkrgkMXdvIsVRRTs69DWYA8uNNQsu1stfVKU=
github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E=
go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=

View file

@ -2473,14 +2473,14 @@ func BitmapsToRoaring(bitmaps []*Bitmap) []byte {
nextData := data[dataOffset:]
switch c.typeID { // TODO: make this work on big endian machines
case ContainerArray:
dataOffset += 2 * copy((*[1 << 16]uint16)(unsafe.Pointer(&nextData[0]))[:], c.array())
dataOffset += 2 * copy((*[1 << 16]uint16)(unsafe.Pointer(&nextData[0]))[:c.n:c.n], c.array())
case ContainerBitmap:
copy((*[1024]uint64)(unsafe.Pointer(&nextData[0]))[:], c.bitmap())
dataOffset += 8192
case ContainerRun:
binary.LittleEndian.PutUint16(nextData[0:2], uint16(c.len))
dataOffset += 2
dataOffset += 4 * copy((*[1 << 15]Interval16)(unsafe.Pointer(&nextData[2]))[:], c.runs())
dataOffset += 4 * copy((*[1 << 15]Interval16)(unsafe.Pointer(&nextData[2]))[:c.len:c.len], c.runs())
}
}
}