diff --git a/Makefile b/Makefile index 058a88847..3f1a42949 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,6 @@ LDFLAGS="-X github.com/pilosa/pilosa/v2.Version=$(VERSION) -X github.com/pilosa/ GO_VERSION=latest RELEASE ?= 0 RELEASE_ENABLED = $(subst 0,,$(RELEASE)) -NOCHECKPTR=$(shell go version | grep -q 'go1.1[4,5,6,7]' && echo \"-gcflags=all=-d=checkptr=0\" ) BUILD_TAGS += $(if $(RELEASE_ENABLED),release) BUILD_TAGS += shardwidth$(SHARD_WIDTH) TEST_TAGS = roaringparanoia @@ -39,11 +38,11 @@ vendor: go.mod # Run test suite test: - go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v + go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v # Run test suite with race flag test-race: - go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race $(NOCHECKPTR) -timeout 60m -v + go test ./... -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -race -timeout 60m -v testv: topt testvsub @@ -58,7 +57,7 @@ testvsub: set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i"; \ cd $$i; pwd; \ - go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -timeout 60m || break; \ + go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -timeout 60m || break; \ echo; echo "999 done testing subpkg $$i"; \ cd ..; \ done @@ -67,7 +66,7 @@ testvsub-race: set -e; for i in boltdb ctl http pg pql rbf roaring server sql txkey; do \ echo; echo "___ testing subpkg $$i -race"; \ cd $$i; pwd; \ - go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -v -race -timeout 60m || break; \ + go test -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -v -race -timeout 60m || break; \ echo; echo "999 done testing subpkg $$i -race"; \ cd ..; \ done @@ -212,49 +211,49 @@ docker-test: # The \-\-\- FAIL avoids counting the extra two FAIL strings at then bottom of log.topt. topt: mv log.topt.roar log.topt.roar.prev || true - go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.roar + go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.roar @echo " log.topt.roar green: \c"; cat log.topt.roar | grep PASS |wc -l @echo " log.topt.roar red: \c"; cat log.topt.roar | grep '\-\-\- FAIL' |wc -l topt-badger: mv log.topt.badger log.topt.badger.prev || true - PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger + PILOSA_TXSRC=badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.badger @echo " log.topt.badger green: \c"; cat log.topt.badger | grep PASS |wc -l @echo " log.topt.badger red: \c"; cat log.topt.badger | grep '\-\-\- FAIL' |wc -l topt-badger-race: mv log.topt.badger-race log.topt.badger-race.prev || true - PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.badger-race + PILOSA_TXSRC=badger go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.badger-race @echo " log.topt.badger-race green: \c"; cat log.topt.badger-race | grep PASS |wc -l @echo " log.topt.badger-race red: \c"; cat log.topt.badger-race | grep '\-\-\- FAIL' |wc -l topt-rbf: mv log.topt.rbf log.topt.rbf.prev || true - PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.rbf + PILOSA_TXSRC=rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.rbf @echo " log.topt.rbf green: \c"; cat log.topt.rbf | grep PASS |wc -l @echo " log.topt.rbf red: \c"; cat log.topt.rbf | grep '\-\-\- FAIL' |wc -l topt-rbf-race: mv log.topt.rbf-race log.topt.rbf-race.prev || true - PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) -timeout 120m 2>&1 | tee log.topt.rbf-race + PILOSA_TXSRC=rbf go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) -timeout 120m 2>&1 | tee log.topt.rbf-race @echo " log.topt.rbf-race green: \c"; cat log.topt.rbf-race | grep PASS |wc -l @echo " log.topt.rbf-race red: \c"; cat log.topt.rbf-race | grep '\-\-\- FAIL' |wc -l topt-lmdb: mv log.topt.lmdb log.topt.lmdb.prev || true - PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb + PILOSA_TXSRC=lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lmdb @echo " log.topt.lmdb green: \c"; cat log.topt.lmdb | grep PASS |wc -l @echo " log.topt.lmdb red: \c"; cat log.topt.lmdb | grep '\-\-\- FAIL' |wc -l topt-lmdb-race: mv log.topt.lmdb log.topt.lmdb.prev || true - PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lmdb-race + PILOSA_TXSRC=lmdb go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lmdb-race @echo " log.topt.lmdb-race green: \c"; cat log.topt.lmdb-race | grep PASS |wc -l @echo " log.topt.lmdb-race red: \c"; cat log.topt.lmdb-race | grep '\-\-\- FAIL' |wc -l topt-race: mv log.topt.race log.topt.race.prev || true - go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.race + go test -race -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.race @echo " log.topt.race green: \c"; cat log.topt.race | grep PASS |wc -l @echo " log.topt.race red: \c"; cat log.topt.race | grep '\-\-\- FAIL' |wc -l @@ -263,73 +262,73 @@ topt-race: bg-rr: # shorthand for bluegreen test with A:badger; B:roaring mv log.bg-rr log.bg-rr.prev || true - PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rr + PILOSA_TXSRC=badger_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bg-rr @echo " log.bg-rr green: \c"; cat log.bg-rr | grep PASS |wc -l @echo " log.bg-rr red: \c"; cat log.bg-rr | grep '\-\-\- FAIL' |wc -l rr-bg: # bluegreen with A:roaring; B:badger (B's values are returned). mv log.bg.roar_bg log.bg.roar_bg.prev || true - PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-bg + PILOSA_TXSRC=roaring_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-bg @echo " log.rr-bg green: \c"; cat log.rr-bg | grep PASS |wc -l @echo " log.rr-bg red: \c"; cat log.rr-bg | grep '\-\-\- FAIL' |wc -l rbf-rr: mv log.rbf-rr log.rbf-rr.prev || true - PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-rr + PILOSA_TXSRC=rbf_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-rr @echo " log.rbf-rr green: \c"; cat log.rbf-rr | grep PASS |wc -l @echo " log.rbf-rr red: \c"; cat log.rbf-rr | grep '\-\-\- FAIL' |wc -l rr-rbf: mv log.rr-rbf log.rr-rbf.prev || true - PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-rbf + PILOSA_TXSRC=roaring_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-rbf @echo " log.rr-rbf green: \c"; cat log.rr-rbf | grep PASS |wc -l @echo " log.rr-rbf red: \c"; cat log.rr-rbf | grep '\-\-\- FAIL' |wc -l rbf-bg: mv log.rbf-bg log.rbf-bg.prev || true - PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-bg + PILOSA_TXSRC=rbf_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-bg @echo " log.rbf-bg green: \c"; cat log.rbf-bg | grep PASS |wc -l @echo " log.rbf-bg red: \c"; cat log.rbf-bg | grep '\-\-\- FAIL' |wc -l bg-rbf: mv log.bg-rbf log.bg-rbf.prev || true - PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.bg-rbf + PILOSA_TXSRC=badger_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.bg-rbf @echo " log.bg-rbf green: \c"; cat log.bg-rbf | grep PASS |wc -l @echo " log.bg-rbf red: \c"; cat log.bg-rbf | grep '\-\-\- FAIL' |wc -l rbf-lm: mv log.rbf-lm log.rbf-lm.prev || true - PILOSA_TXSRC=rbf_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rbf-lm + PILOSA_TXSRC=rbf_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rbf-lm @echo " log.rbf-lm green: \c"; cat log.rbf-lm | grep PASS |wc -l @echo " log.rbf-lm red: \c"; cat log.rbf-lm | grep '\-\-\- FAIL' |wc -l lm-rbf: mv log.lm-rbf log.lm-rbf.prev || true - PILOSA_TXSRC=lmdb_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rbf + PILOSA_TXSRC=lmdb_rbf go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.lm-rbf @echo " log.lm-rbf green: \c"; cat log.lm-rbf | grep PASS |wc -l @echo " log.lm-rbf red: \c"; cat log.lm-rbf | grep '\-\-\- FAIL' |wc -l lm-rr: mv log.lm-rr log.lm-rr.prev || true - PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.lm-rr + PILOSA_TXSRC=lmdb_roaring go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.lm-rr @echo " log.lm-rr green: \c"; cat log.lm-rr | grep PASS |wc -l @echo " log.lm-rr red: \c"; cat log.lm-rr | grep '\-\-\- FAIL' |wc -l rr-lm: mv log.rr-lm log.rr-lm.prev || true - PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.rr-lm + PILOSA_TXSRC=roaring_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.rr-lm @echo " log.rr-lm green: \c"; cat log.rr-lm | grep PASS |wc -l @echo " log.rr-lm red: \c"; cat log.rr-lm | grep '\-\-\- FAIL' |wc -l bg-lm: mv log.topt.bg-lm log.topt.bg-lm.prev || true - PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.bg-lm + PILOSA_TXSRC=badger_lmdb go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.bg-lm @echo " log.topt.bg-lm green: \c"; cat log.topt.bg-lm | grep PASS |wc -l @echo " log.topt.bg-lm red: \c"; cat log.topt.bg-lm | grep '\-\-\- FAIL' |wc -l lm-bg: mv log.topt.lm-bg log.topt.lm-bg.prev || true - PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) $(NOCHECKPTR) 2>&1 | tee log.topt.lm-bg + PILOSA_TXSRC=lmdb_badger go test -v -tags='$(BUILD_TAGS) $(TEST_TAGS)' $(TESTFLAGS) 2>&1 | tee log.topt.lm-bg @echo " log.topt.lm-bg green: \c"; cat log.topt.lm-bg | grep PASS |wc -l @echo " log.topt.lm-bg red: \c"; cat log.topt.lm-bg | grep '\-\-\- FAIL' |wc -l diff --git a/boltdb/attrstore.go b/boltdb/attrstore.go index f29cec183..cf5c62473 100644 --- a/boltdb/attrstore.go +++ b/boltdb/attrstore.go @@ -25,9 +25,9 @@ import ( "github.com/cespare/xxhash" - "github.com/boltdb/bolt" "github.com/pilosa/pilosa/v2" "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" ) // attrBlockSize is the size of attribute blocks for anti-entropy. diff --git a/boltdb/translate.go b/boltdb/translate.go index b5a0e5651..472645367 100644 --- a/boltdb/translate.go +++ b/boltdb/translate.go @@ -26,10 +26,10 @@ import ( "sync" "time" - "github.com/boltdb/bolt" "github.com/pilosa/pilosa/v2" "github.com/pkg/errors" "github.com/zeebo/blake3" + bolt "go.etcd.io/bbolt" "runtime/pprof" ) diff --git a/go.mod b/go.mod index 7ffb4b20f..747138632 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/DataDog/datadog-go v0.0.0-20180822151419-281ae9f2d895 github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/benbjohnson/immutable v0.2.0 - github.com/boltdb/bolt v1.3.1 github.com/cespare/xxhash v1.1.0 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect github.com/davecgh/go-spew v1.1.1 @@ -41,6 +40,7 @@ require ( github.com/uber/jaeger-client-go v2.16.0+incompatible github.com/uber/jaeger-lib v2.2.0+incompatible // indirect github.com/zeebo/blake3 v0.0.4 + go.etcd.io/bbolt v1.3.5 golang.org/x/mod v0.3.0 golang.org/x/net v0.0.0-20200822124328-c89045814202 // indirect golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 diff --git a/go.sum b/go.sum index 2f6d5c213..9a94934aa 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,6 @@ github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylH github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= -github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= @@ -258,7 +256,10 @@ github.com/zeebo/blake3 v0.0.4 h1:vtZ4X8B2lKXZFg2Xyg6Wo36mvmnJvc2VQYTtA4RDCkI= github.com/zeebo/blake3 v0.0.4/go.mod h1:YOZo8A49yNqM0X/Y+JmDUZshJWLt1laHsNSn5ny2i34= github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05 h1:4pW5fMvVkrgkMXdvIsVRRTs69DWYA8uNNQsu1stfVKU= github.com/zeebo/pcg v0.0.0-20181207190024-3cdc6b625a05/go.mod h1:Gr+78ptB0MwXxm//LBaEvBiaXY7hXJ6KGe2V32X2F6E= +go.etcd.io/bbolt v1.3.2 h1:Z/90sZLPOeCy2PwprqkFa25PdkusRzaj9P8zm/KNyvk= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= diff --git a/roaring/roaring.go b/roaring/roaring.go index d427def14..8cf2587a5 100644 --- a/roaring/roaring.go +++ b/roaring/roaring.go @@ -2473,14 +2473,14 @@ func BitmapsToRoaring(bitmaps []*Bitmap) []byte { nextData := data[dataOffset:] switch c.typeID { // TODO: make this work on big endian machines case ContainerArray: - dataOffset += 2 * copy((*[1 << 16]uint16)(unsafe.Pointer(&nextData[0]))[:], c.array()) + dataOffset += 2 * copy((*[1 << 16]uint16)(unsafe.Pointer(&nextData[0]))[:c.n:c.n], c.array()) case ContainerBitmap: copy((*[1024]uint64)(unsafe.Pointer(&nextData[0]))[:], c.bitmap()) dataOffset += 8192 case ContainerRun: binary.LittleEndian.PutUint16(nextData[0:2], uint16(c.len)) dataOffset += 2 - dataOffset += 4 * copy((*[1 << 15]Interval16)(unsafe.Pointer(&nextData[2]))[:], c.runs()) + dataOffset += 4 * copy((*[1 << 15]Interval16)(unsafe.Pointer(&nextData[2]))[:c.len:c.len], c.runs()) } } }